From 3d39434c36bcc7cd1da4e4885829a85077616aa3 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 27 Jul 2015 19:31:45 +0000 Subject: [PATCH 001/179] Add an experimental "BEGIN UNLOCKED" command. FossilOrigin-Name: 8079421604dbd40d03471dad6d12115119b554c2 --- manifest | 47 +++---- manifest.uuid | 2 +- src/btree.c | 30 ++++- src/btree.h | 1 + src/build.c | 4 +- src/main.c | 1 + src/pager.c | 64 ++++++++- src/pager.h | 5 +- src/parse.y | 1 + src/sqliteInt.h | 1 + src/vacuum.c | 1 + src/vdbe.c | 36 ++++- src/vdbeaux.c | 28 +++- src/wal.c | 206 ++++++++++++++++++++++++----- src/wal.h | 5 + test/unlocked.test | 307 +++++++++++++++++++++++++++++++++++++++++++ tool/mkkeywordhash.c | 1 + 17 files changed, 658 insertions(+), 82 deletions(-) create mode 100644 test/unlocked.test diff --git a/manifest b/manifest index 9db7a69587..62addb80e6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Version\s3.8.11 -D 2015-07-27T13:49:41.754 +C Add\san\sexperimental\s"BEGIN\sUNLOCKED"\scommand. +D 2015-07-27T19:31:45.102 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 4de3ef40c8b3b75c0c55ff4242a43c8ce1ad90ee F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -270,10 +270,10 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 4d9134dc988a87838c06056c89c0e8c4700a0452 F src/bitvec.c d1f21d7d91690747881f03940584f4cc548c9d3d F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 -F src/btree.c f48b3ef91676c06a90a8832987ecef6b94c931ee -F src/btree.h 969adc948e89e449220ff0ff724c94bb2a52e9f1 +F src/btree.c b184ce6d483dfadc53126a4b7994633ec0d5ef5f +F src/btree.h 550e8447684013a69196ec664142f67538da8ef1 F src/btreeInt.h 2ad754dd4528baa8d0946a593cc373b890bf859e -F src/build.c b3f15255d5b16e42dafeaa638fd4f8a47c94ed70 +F src/build.c 28c15c43eefc0066ff64040526ff649c32fe5523 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b @@ -293,7 +293,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 92bafa308607dd985ca389a788cd9e0a2b608712 F src/loadext.c dfcee8c7c032cd0fd55af3e0fc1fcfb01e426df2 -F src/main.c 0a60b7ca8252c3a6f95438fa4ce8fe5b275c69f2 +F src/main.c ebbd5a45caf66ce0ebdf1b6c0bc4784e3cfcd768 F src/malloc.c 19461e159bccf0e2cf06a50e867963d0a7b124a8 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c abe6ee469b6c5a35c7f22bfeb9c9bac664a1c987 @@ -315,9 +315,9 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 388c023582b17890f10c980b30ec1922b471753b F src/os_win.c 40b3af7a47eb1107d0d69e592bec345a3b7b798a F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c aa916ca28606ccf4b6877dfc2b643ccbca86589f -F src/pager.h 6d435f563b3f7fcae4b84433b76a6ac2730036e2 -F src/parse.y 6d60dda8f8d418b6dc034f1fbccd816c459983a8 +F src/pager.c c9f22eb831851220c1095863c5797a88f71bea8a +F src/pager.h b6cf6e9552c5b1e7d6d0d8cd7ca51be7802bec72 +F src/parse.y 199145cd982587d70fa3db2f9b11f7ab118f0acd F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0 F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9 F src/pcache1.c d08939800abf3031bd0affd5a13fbc4d7ba3fb68 @@ -333,7 +333,7 @@ F src/shell.c 8af3cced094aebb5f57a8ad739b9dafc7867eed7 F src/sqlite.h.in f623dd30a4fb7df2fb44a2a85e27813d25e486c2 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h a0b948ebc89bac13941254641326a6aa248c2cc4 -F src/sqliteInt.h 81f458941cfb30c5536c37930fb6f41e66171284 +F src/sqliteInt.h f14033fa6da3ca70f7dde5ba19dc1ed58055a2d7 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e @@ -390,20 +390,20 @@ F src/trigger.c 322f23aad694e8f31d384dcfa386d52a48d3c52f F src/update.c 487747b328b7216bb7f6af0695d6937d5c9e605f F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 46358a204b35971a839341cf64599d65b151ba88 -F src/vacuum.c 2ddd5cad2a7b9cef7f9e431b8c7771634c6b1701 -F src/vdbe.c 86be40a4568f5385048ea6623adfd11ddb7bf710 +F src/vacuum.c 4e7e18d889662eda281855a7125e56269069f215 +F src/vdbe.c fd6832408d9621ad347a65b01a05380fe766232f F src/vdbe.h 7a75045d879118b9d3af7e8b3c108f2f27c51473 F src/vdbeInt.h 8b54e01ad0463590e7cffabce0bc36da9ee4f816 F src/vdbeapi.c adabbd66eb2e3a10f3998485ee0be7e326d06ee4 -F src/vdbeaux.c 787f5f9d58f4c6f39294ed06909ba602d1a402e6 +F src/vdbeaux.c 27feae106f3f4db298e65d37a9da4962aaff724b F src/vdbeblob.c 4f2e8e075d238392df98c5e03a64342465b03f90 F src/vdbemem.c ae38a0d35ae71cf604381a887c170466ba518090 F src/vdbesort.c f5009e7a35e3065635d8918b9a31f498a499976b F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 F src/vtab.c 082b35a25a26e3d36f365ca8cd73c1922532f05e F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb -F src/wal.c 6fb6b68969e4692593c2552c4e7bff5882de2cb8 -F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 +F src/wal.c d588e26ed0d912584551dfe65d644d8c27cc2d1f +F src/wal.h 5188cd85398dc612ae5ed2ed02d193377218be56 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c 909eba3b6db984eb2adfbca9de2c237ee7056adb F src/whereInt.h 5f87e3c4b0551747d119730dfebddd3c54f04047 @@ -1215,6 +1215,7 @@ F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unique2.test 41e7f83c6827605991160a31380148a9fc5f1339 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 +F test/unlocked.test edd4ed073ab16c0f8c9f5124e039100657b422c9 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b @@ -1326,7 +1327,7 @@ F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh d1a2da0e15b2ed33d60af35c7e9d483f13a8eb9f -F tool/mkkeywordhash.c dfff09dbbfaf950e89af294f48f902181b144670 +F tool/mkkeywordhash.c de4a823fe66f9e8c39c86e465ac7285fd6935bb8 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkpragmatab.tcl 84af2b180484323a2ea22a2279e8bd9e3e1e492e F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 @@ -1366,10 +1367,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9f1dcdc3e1644c21762dacf619fac70afe6318c5 -R 8f58f88db8a7564cc507b9707cc8c853 -T +bgcolor * #d0c0ff -T +sym-release * -T +sym-version-3.8.11 * -U drh -Z ff777afb71751a2633799d5aca45576d +P b8e92227a469de677a66da62e4361f099c0b79d0 +R 889966cd4fab23d4f4355c880be4edd8 +T *branch * experimental-begin-unlocked +T *sym-experimental-begin-unlocked * +T -sym-trunk * +U dan +Z 55f5e7a6a42bdc95bbb980b319cd8658 diff --git a/manifest.uuid b/manifest.uuid index 8247151f16..25c9006198 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b8e92227a469de677a66da62e4361f099c0b79d0 \ No newline at end of file +8079421604dbd40d03471dad6d12115119b554c2 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 2692ade8e4..c83a01ad4c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3142,7 +3142,10 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ if( (pBt->btsFlags & BTS_READ_ONLY)!=0 ){ rc = SQLITE_READONLY; }else{ - rc = sqlite3PagerBegin(pBt->pPager,wrflag>1,sqlite3TempInMemory(p->db)); + int bSubjInMem = sqlite3TempInMemory(p->db); + int exFlag = p->db->bUnlocked ? -1 : (wrflag>1); + assert( p->db->bUnlocked==0 || wrflag==1 ); + rc = sqlite3PagerBegin(pBt->pPager, exFlag, bSubjInMem); if( rc==SQLITE_OK ){ rc = newDatabase(pBt); } @@ -3670,8 +3673,15 @@ int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zMaster){ if( p->inTrans==TRANS_WRITE ){ BtShared *pBt = p->pBt; sqlite3BtreeEnter(p); + #ifndef SQLITE_OMIT_AUTOVACUUM - if( pBt->autoVacuum ){ + /* Figure out if this is a commit of an UNLOCKED transaction that + ** requires a snapshot upgrade. If so, skip any auto-vacuum + ** processing. */ + if( pBt->autoVacuum && ( + 0==pBt->db->bUnlocked + || 0==sqlite3PagerCommitRequiresUpgrade(pBt->pPager) + )){ rc = autoVacuumCommit(pBt); if( rc!=SQLITE_OK ){ sqlite3BtreeLeave(p); @@ -3682,7 +3692,9 @@ int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zMaster){ sqlite3PagerTruncateImage(pBt->pPager, pBt->nPage); } #endif - rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zMaster, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zMaster, 0); + } sqlite3BtreeLeave(p); } return rc; @@ -9578,3 +9590,15 @@ int sqlite3BtreeIsReadonly(Btree *p){ ** Return the size of the header added to each page by this module. */ int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); } + +int sqlite3BtreeExclusiveLock(Btree *p){ + int rc; + BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); + rc = sqlite3PagerExclusiveLock(pBt->pPager, pBt->pPage1->pDbPage); + sqlite3BtreeLeave(p); + return rc; +} + + + diff --git a/src/btree.h b/src/btree.h index 3edc2b3b57..ae9b2c4e59 100644 --- a/src/btree.h +++ b/src/btree.h @@ -270,5 +270,6 @@ void sqlite3BtreeCursorList(Btree*); # define sqlite3SchemaMutexHeld(X,Y,Z) 1 #endif +int sqlite3BtreeExclusiveLock(Btree*); #endif /* _BTREE_H_ */ diff --git a/src/build.c b/src/build.c index 2936805365..0300e2fc15 100644 --- a/src/build.c +++ b/src/build.c @@ -3828,13 +3828,13 @@ void sqlite3BeginTransaction(Parse *pParse, int type){ } v = sqlite3GetVdbe(pParse); if( !v ) return; - if( type!=TK_DEFERRED ){ + if( type==TK_IMMEDIATE || type==TK_EXCLUSIVE ){ for(i=0; inDb; i++){ sqlite3VdbeAddOp2(v, OP_Transaction, i, (type==TK_EXCLUSIVE)+1); sqlite3VdbeUsesBtree(v, i); } } - sqlite3VdbeAddOp2(v, OP_AutoCommit, 0, 0); + sqlite3VdbeAddOp3(v, OP_AutoCommit, 0, 0, (type==TK_UNLOCKED)); } /* diff --git a/src/main.c b/src/main.c index 36206eec8c..27bb2f0e83 100644 --- a/src/main.c +++ b/src/main.c @@ -2737,6 +2737,7 @@ static int openDatabase( memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = SQLITE_DEFAULT_WORKER_THREADS; db->autoCommit = 1; + db->bUnlocked = 0; db->nextAutovac = -1; db->szMmap = sqlite3GlobalConfig.szMmap; db->nextPagesize = 0; diff --git a/src/pager.c b/src/pager.c index 060edb8d1d..2ea3b90a96 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3066,6 +3066,7 @@ static int pagerWalFrames( ** list here. */ PgHdr **ppNext = &pList; nList = 0; + for(p=pList; (*ppNext = p)!=0; p=p->pDirty){ if( p->pgno<=nTruncate ){ ppNext = &p->pDirty; @@ -4081,7 +4082,7 @@ static int syncJournal(Pager *pPager, int newHdr){ assert( assert_pager_state(pPager) ); assert( !pagerUseWal(pPager) ); - rc = sqlite3PagerExclusiveLock(pPager); + rc = sqlite3PagerExclusiveLock(pPager, 0); if( rc!=SQLITE_OK ) return rc; if( !pPager->noSync ){ @@ -4429,6 +4430,10 @@ static int pagerStress(void *p, PgHdr *pPg){ pPg->pDirty = 0; if( pagerUseWal(pPager) ){ + /* If the transaction is a "BEGIN UNLOCKED" transaction, the page + ** cannot be flushed to disk. Return early in this case. */ + if( sqlite3WalIsInTrans(pPager->pWal)==0 ) return SQLITE_OK; + /* Write a single frame for this page to the log. */ rc = subjournalPageIfRequired(pPg); if( rc==SQLITE_OK ){ @@ -5548,11 +5553,14 @@ static int pager_open_journal(Pager *pPager){ ** Begin a write-transaction on the specified pager object. If a ** write-transaction has already been opened, this function is a no-op. ** -** If the exFlag argument is false, then acquire at least a RESERVED -** lock on the database file. If exFlag is true, then acquire at least +** If the exFlag argument is 0, then acquire at least a RESERVED +** lock on the database file. If exFlag is >0, then acquire at least ** an EXCLUSIVE lock. If such a lock is already held, no locking ** functions need be called. ** +** If (exFlag<0) and the database is in WAL mode, do not take any locks. +** The transaction will run in UNLOCKED mode instead. +** ** If the subjInMemory argument is non-zero, then any sub-journal opened ** within this transaction will be opened as an in-memory file. This ** has no effect if the sub-journal is already opened (as it may be when @@ -5588,7 +5596,9 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ ** The busy-handler is not invoked if another connection already ** holds the write-lock. If possible, the upper layer will call it. */ - rc = sqlite3WalBeginWriteTransaction(pPager->pWal); + if( exFlag>=0 ){ + rc = sqlite3WalBeginWriteTransaction(pPager->pWal); + } }else{ /* Obtain a RESERVED lock on the database file. If the exFlag parameter ** is true, then immediately upgrade this to an EXCLUSIVE lock. The @@ -5596,7 +5606,7 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ ** lock, but not when obtaining the RESERVED lock. */ rc = pagerLockDb(pPager, RESERVED_LOCK); - if( rc==SQLITE_OK && exFlag ){ + if( rc==SQLITE_OK && exFlag>0 ){ rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); } } @@ -6056,7 +6066,7 @@ int sqlite3PagerSync(Pager *pPager, const char *zMaster){ ** Otherwise, either SQLITE_BUSY or an SQLITE_IOERR_XXX error code is ** returned. */ -int sqlite3PagerExclusiveLock(Pager *pPager){ +int sqlite3PagerExclusiveLock(Pager *pPager, PgHdr *pPage1){ int rc = SQLITE_OK; assert( pPager->eState==PAGER_WRITER_CACHEMOD || pPager->eState==PAGER_WRITER_DBMOD @@ -6065,10 +6075,52 @@ int sqlite3PagerExclusiveLock(Pager *pPager){ assert( assert_pager_state(pPager) ); if( 0==pagerUseWal(pPager) ){ rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + }else{ + Wal *pWal = pPager->pWal; + if( 0==sqlite3WalIsInTrans(pWal) ){ + /* TODO: There must be an optimization opportunity here, as this call + ** to PcacheDirtyList() sorts the list of dirty pages, even though it + ** is not really required - and will be sorted again in CommitPhaseOne() + ** in any case. */ + PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); + + /* This is an UNLOCKED transaction. Attempt to lock the wal database + ** here. If SQLITE_BUSY (but not SQLITE_BUSY_SNAPSHOT) is returned, + ** invoke the busy-handler and try again for as long as it returns + ** non-zero. */ + do { + /* rc = sqlite3WalBeginWriteTransaction(pWal); */ + rc = sqlite3WalLockForCommit(pWal, pList, pPage1); + }while( rc==SQLITE_BUSY + && pPager->xBusyHandler(pPager->pBusyHandlerArg) + ); + } } return rc; } +/* +** If this is a WAL mode connection and the WRITER lock is currently held, +** relinquish it. +*/ +void sqlite3PagerDropExclusiveLock(Pager *pPager){ + if( pagerUseWal(pPager) ){ + sqlite3WalEndWriteTransaction(pPager->pWal); + } +} + +/* +** Return true if this is a WAL database and snapshot upgrade is required +** before the current transaction can be committed. +*/ +int sqlite3PagerCommitRequiresUpgrade(Pager *pPager){ + int res = 0; + if( pagerUseWal(pPager) ){ + res = sqlite3WalCommitRequiresUpgrade(pPager->pWal); + } + return res; +} + /* ** Sync the database file for the pager pPager. zMaster points to the name ** of a master journal file that should be written into the individual diff --git a/src/pager.h b/src/pager.h index e3b57f435e..57d2812701 100644 --- a/src/pager.h +++ b/src/pager.h @@ -150,7 +150,7 @@ void *sqlite3PagerGetExtra(DbPage *); void sqlite3PagerPagecount(Pager*, int*); int sqlite3PagerBegin(Pager*, int exFlag, int); int sqlite3PagerCommitPhaseOne(Pager*,const char *zMaster, int); -int sqlite3PagerExclusiveLock(Pager*); +int sqlite3PagerExclusiveLock(Pager*, DbPage *pPage1); int sqlite3PagerSync(Pager *pPager, const char *zMaster); int sqlite3PagerCommitPhaseTwo(Pager*); int sqlite3PagerRollback(Pager*); @@ -158,6 +158,9 @@ int sqlite3PagerOpenSavepoint(Pager *pPager, int n); int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint); int sqlite3PagerSharedLock(Pager *pPager); +void sqlite3PagerDropExclusiveLock(Pager*); +int sqlite3PagerCommitRequiresUpgrade(Pager*); + #ifndef SQLITE_OMIT_WAL int sqlite3PagerCheckpoint(Pager *pPager, int, int*, int*); int sqlite3PagerWalSupported(Pager *pPager); diff --git a/src/parse.y b/src/parse.y index d7aa763683..db40b70b9c 100644 --- a/src/parse.y +++ b/src/parse.y @@ -121,6 +121,7 @@ transtype(A) ::= . {A = TK_DEFERRED;} transtype(A) ::= DEFERRED(X). {A = @X;} transtype(A) ::= IMMEDIATE(X). {A = @X;} transtype(A) ::= EXCLUSIVE(X). {A = @X;} +transtype(A) ::= UNLOCKED(X). {A = @X;} cmd ::= COMMIT trans_opt. {sqlite3CommitTransaction(pParse);} cmd ::= END trans_opt. {sqlite3CommitTransaction(pParse);} cmd ::= ROLLBACK trans_opt. {sqlite3RollbackTransaction(pParse);} diff --git a/src/sqliteInt.h b/src/sqliteInt.h index d26cd19eb7..2480bb0cfe 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1147,6 +1147,7 @@ struct sqlite3 { u16 dbOptFlags; /* Flags to enable/disable optimizations */ u8 enc; /* Text encoding */ u8 autoCommit; /* The auto-commit flag. */ + u8 bUnlocked; /* Current transaction is "UNLOCKED" */ u8 temp_store; /* 1: file 2: memory 0: default */ u8 mallocFailed; /* True if we have seen a malloc failure */ u8 dfltLockMode; /* Default locking-mode for attached dbs */ diff --git a/src/vacuum.c b/src/vacuum.c index adc802e60b..10f14cbf47 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -356,6 +356,7 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ ** is closed by the DETACH. */ db->autoCommit = 1; + db->bUnlocked = 0; if( pDb ){ sqlite3BtreeClose(pDb->pBt); diff --git a/src/vdbe.c b/src/vdbe.c index fe97087c09..b400c425b8 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2899,6 +2899,7 @@ case OP_Savepoint: { if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ goto vdbe_return; } + assert( db->bUnlocked==0 ); db->autoCommit = 1; if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){ p->pc = (int)(pOp - aOp); @@ -2970,29 +2971,39 @@ case OP_Savepoint: { break; } -/* Opcode: AutoCommit P1 P2 * * * +/* Opcode: AutoCommit P1 P2 P3 * * ** ** Set the database auto-commit flag to P1 (1 or 0). If P2 is true, roll ** back any currently active btree transactions. If there are any active ** VMs (apart from this one), then a ROLLBACK fails. A COMMIT fails if ** there are active writing VMs or active VMs that use shared cache. ** +** If P3 is non-zero, then this instruction is being executed as part of +** a "BEGIN UNLOCKED" command. +** ** This instruction causes the VM to halt. */ case OP_AutoCommit: { int desiredAutoCommit; int iRollback; int turnOnAC; + int bUnlocked; + int hrc; desiredAutoCommit = pOp->p1; iRollback = pOp->p2; + bUnlocked = pOp->p3; turnOnAC = desiredAutoCommit && !db->autoCommit; assert( desiredAutoCommit==1 || desiredAutoCommit==0 ); assert( desiredAutoCommit==1 || iRollback==0 ); + assert( desiredAutoCommit==0 || bUnlocked==0 ); + assert( db->autoCommit==0 || db->bUnlocked==0 ); assert( db->nVdbeActive>0 ); /* At least this one VM is active */ assert( p->bIsReader ); - if( turnOnAC && !iRollback && db->nVdbeWrite>0 ){ + if( turnOnAC && !iRollback && + (db->nVdbeWrite>0 || (db->bUnlocked && db->nVdbeActive>1)) + ){ /* If this instruction implements a COMMIT and other VMs are writing ** return an error indicating that the other VMs must complete first. */ @@ -3004,16 +3015,20 @@ case OP_AutoCommit: { assert( desiredAutoCommit==1 ); sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); db->autoCommit = 1; + db->bUnlocked = 0; }else if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ goto vdbe_return; }else{ db->autoCommit = (u8)desiredAutoCommit; - if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){ + hrc = sqlite3VdbeHalt(p); + if( (hrc & 0xFF)==SQLITE_BUSY ){ p->pc = (int)(pOp - aOp); db->autoCommit = (u8)(1-desiredAutoCommit); - p->rc = rc = SQLITE_BUSY; + p->rc = hrc; + rc = SQLITE_BUSY; goto vdbe_return; } + db->bUnlocked = (u8)bUnlocked; } assert( db->nStatement==0 ); sqlite3CloseSavepoints(db); @@ -3208,9 +3223,16 @@ case OP_SetCookie: { /* in3 */ /* See note about index shifting on OP_ReadCookie */ rc = sqlite3BtreeUpdateMeta(pDb->pBt, pOp->p2, (int)pIn3->u.i); if( pOp->p2==BTREE_SCHEMA_VERSION ){ - /* When the schema cookie changes, record the new cookie internally */ - pDb->pSchema->schema_cookie = (int)pIn3->u.i; - db->flags |= SQLITE_InternChanges; + if( db->bUnlocked ){ + sqlite3VdbeError(p, "cannot modify database schema - " + "UNLOCKED transaction" + ); + rc = SQLITE_ERROR; + }else{ + /* When the schema cookie changes, record the new cookie internally */ + pDb->pSchema->schema_cookie = (int)pIn3->u.i; + db->flags |= SQLITE_InternChanges; + } }else if( pOp->p2==BTREE_FILE_FORMAT ){ /* Record changes in the file format */ pDb->pSchema->file_format = (u8)pIn3->u.i; diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 08dc885df6..f5f9d0fbdd 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2020,11 +2020,24 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ if( sqlite3BtreeIsInTrans(pBt) ){ needXcommit = 1; if( i!=1 ) nTrans++; - sqlite3BtreeEnter(pBt); - rc = sqlite3PagerExclusiveLock(sqlite3BtreePager(pBt)); - sqlite3BtreeLeave(pBt); + rc = sqlite3BtreeExclusiveLock(pBt); } } + + if( db->bUnlocked && (rc & 0xFF)==SQLITE_BUSY ){ + /* An SQLITE_BUSY or SQLITE_BUSY_SNAPSHOT was encountered while + ** attempting to take the WRITER lock on a wal file. Release the + ** WRITER locks on all wal files and return early. */ + for(i=0; inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( sqlite3BtreeIsInTrans(pBt) ){ + sqlite3BtreeEnter(pBt); + sqlite3PagerDropExclusiveLock(sqlite3BtreePager(pBt)); + sqlite3BtreeLeave(pBt); + } + } + } + if( rc!=SQLITE_OK ){ return rc; } @@ -2427,6 +2440,7 @@ int sqlite3VdbeHalt(Vdbe *p){ sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; + db->bUnlocked = 0; p->nChange = 0; } } @@ -2462,9 +2476,9 @@ int sqlite3VdbeHalt(Vdbe *p){ ** is required. */ rc = vdbeCommit(db, p); } - if( rc==SQLITE_BUSY && p->readOnly ){ + if( (rc & 0xFF)==SQLITE_BUSY && p->readOnly ){ sqlite3VdbeLeave(p); - return SQLITE_BUSY; + return rc; }else if( rc!=SQLITE_OK ){ p->rc = rc; sqlite3RollbackAll(db, SQLITE_OK); @@ -2489,6 +2503,7 @@ int sqlite3VdbeHalt(Vdbe *p){ sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; + db->bUnlocked = 0; p->nChange = 0; } } @@ -2510,6 +2525,7 @@ int sqlite3VdbeHalt(Vdbe *p){ sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; + db->bUnlocked = 0; p->nChange = 0; } } @@ -2554,7 +2570,7 @@ int sqlite3VdbeHalt(Vdbe *p){ } assert( db->nVdbeActive>0 || db->autoCommit==0 || db->nStatement==0 ); - return (p->rc==SQLITE_BUSY ? SQLITE_BUSY : SQLITE_OK); + return ((p->rc & 0xFF)==SQLITE_BUSY ? SQLITE_BUSY : SQLITE_OK); } diff --git a/src/wal.c b/src/wal.c index f7e2594001..a2200e7005 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2357,38 +2357,19 @@ void sqlite3WalEndReadTransaction(Wal *pWal){ } /* -** Search the wal file for page pgno. If found, set *piRead to the frame that -** contains the page. Otherwise, if pgno is not in the wal file, set *piRead -** to zero. -** -** Return SQLITE_OK if successful, or an error code if an error occurs. If an -** error does occur, the final value of *piRead is undefined. -*/ -int sqlite3WalFindFrame( - Wal *pWal, /* WAL handle */ - Pgno pgno, /* Database page number to read data for */ - u32 *piRead /* OUT: Frame number (or zero) */ +** Search the hash tables for an entry matching page number pgno. Ignore +** any entries that lie after frame iLast within the wal file. +*/ +static int walFindFrame( + Wal *pWal, + Pgno pgno, + u32 iLast, + u32 *piRead ){ - u32 iRead = 0; /* If !=0, WAL frame to return data from */ - u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ int iHash; /* Used to loop through N hash tables */ + u32 iRead = 0; - /* This routine is only be called from within a read transaction. */ - assert( pWal->readLock>=0 || pWal->lockError ); - - /* If the "last page" field of the wal-index header snapshot is 0, then - ** no data will be read from the wal under any circumstances. Return early - ** in this case as an optimization. Likewise, if pWal->readLock==0, - ** then the WAL is ignored by the reader so return early, as if the - ** WAL were empty. - */ - if( iLast==0 || pWal->readLock==0 ){ - *piRead = 0; - return SQLITE_OK; - } - - /* Search the hash table or tables for an entry matching page number - ** pgno. Each iteration of the following for() loop searches one + /* Each iteration of the following for() loop searches one ** hash table (each hash table indexes up to HASHTABLE_NPAGE frames). ** ** This code might run concurrently to the code in walIndexAppend() @@ -2437,11 +2418,48 @@ int sqlite3WalFindFrame( } } + *piRead = iRead; + return SQLITE_OK; +} + +/* +** Search the wal file for page pgno. If found, set *piRead to the frame that +** contains the page. Otherwise, if pgno is not in the wal file, set *piRead +** to zero. +** +** Return SQLITE_OK if successful, or an error code if an error occurs. If an +** error does occur, the final value of *piRead is undefined. +*/ +int sqlite3WalFindFrame( + Wal *pWal, /* WAL handle */ + Pgno pgno, /* Database page number to read data for */ + u32 *piRead /* OUT: Frame number (or zero) */ +){ + u32 iRead = 0; /* If !=0, WAL frame to return data from */ + u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ + int rc; + + /* This routine is only be called from within a read transaction. */ + assert( pWal->readLock>=0 || pWal->lockError ); + + /* If the "last page" field of the wal-index header snapshot is 0, then + ** no data will be read from the wal under any circumstances. Return early + ** in this case as an optimization. Likewise, if pWal->readLock==0, + ** then the WAL is ignored by the reader so return early, as if the + ** WAL were empty. + */ + if( iLast==0 || pWal->readLock==0 ){ + *piRead = 0; + return SQLITE_OK; + } + + rc = walFindFrame(pWal, pgno, iLast, &iRead); + #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT /* If expensive assert() statements are available, do a linear search ** of the wal-index file content. Make sure the results agree with the ** result obtained using the hash indexes above. */ - { + if( rc==SQLITE_OK ){ u32 iRead2 = 0; u32 iTest; for(iTest=iLast; iTest>0; iTest--){ @@ -2537,6 +2555,96 @@ int sqlite3WalBeginWriteTransaction(Wal *pWal){ return rc; } +/* +** TODO: Combine some code with BeginWriteTransaction() +** +** This function is only ever called when committing a "BEGIN UNLOCKED" +** transaction. It may be assumed that no frames have been written to +** the wal file. +*/ +int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pList, PgHdr *pPage1){ + volatile WalIndexHdr *pHead; /* Head of the wal file */ + int rc; + + /* Cannot start a write transaction without first holding a read + ** transaction. */ + assert( pWal->readLock>=0 ); + + if( pWal->readOnly ){ + return SQLITE_READONLY; + } + + /* Only one writer allowed at a time. Get the write lock. Return + ** SQLITE_BUSY if unable. + */ + rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1, 0); + if( rc ){ + return rc; + } + pWal->writeLock = 1; + + /* If the database has been modified since this transaction was started, + ** check if it is still possible to commit. The transaction can be + ** committed if: + ** + ** a) None of the pages in pList have been modified since the + ** transaction opened, and + ** + ** b) The database schema cookie has not been modified since the + ** transaction was started. + */ + pHead = walIndexHdr(pWal); + if( memcmp(&pWal->hdr, (void*)pHead, sizeof(WalIndexHdr))!=0 ){ + /* TODO: Is this safe? Because it holds the WRITER lock this thread + ** has exclusive access to the live header, but might it be corrupt? */ + PgHdr *pPg; + u32 iLast = pHead->mxFrame; + for(pPg=pList; rc==SQLITE_OK && pPg; pPg=pPg->pDirty){ + u32 iSlot = 0; + rc = walFindFrame(pWal, pPg->pgno, iLast, &iSlot); + if( iSlot>pWal->hdr.mxFrame ){ + sqlite3_log(SQLITE_OK, + "cannot commit UNLOCKED transaction (conflict at page %d)", + (int)pPg->pgno + ); + rc = SQLITE_BUSY_SNAPSHOT; + } + } + + if( rc==SQLITE_OK ){ + /* Read the newest schema cookie from the wal file. */ + u32 iSlot = 0; + rc = walFindFrame(pWal, 1, iLast, &iSlot); + if( rc==SQLITE_OK && iSlot>pWal->hdr.mxFrame ){ + u8 aNew[4]; + u8 *aOld = &((u8*)pPage1->pData)[40]; + int sz; + i64 iOffset; + sz = pWal->hdr.szPage; + sz = (sz&0xfe00) + ((sz&0x0001)<<16); + iOffset = walFrameOffset(iSlot, sz) + WAL_FRAME_HDRSIZE + 40; + rc = sqlite3OsRead(pWal->pWalFd, aNew, sizeof(aNew), iOffset); + if( rc==SQLITE_OK && memcmp(aOld, aNew, sizeof(aNew)) ){ + /* TODO: New error code? SQLITE_BUSY_SCHEMA. */ + rc = SQLITE_BUSY_SNAPSHOT; + } + } + } + } + + return rc; +} + +/* +** The caller holds the WRITER lock. This function returns true if a snapshot +** upgrade is required before the transaction can be committed, or false +** otherwise. +*/ +int sqlite3WalCommitRequiresUpgrade(Wal *pWal){ + assert( pWal->writeLock ); + return memcmp(&pWal->hdr, (void*)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0; +} + /* ** End a write transaction. The commit has already been done. This ** routine merely releases the lock. @@ -2564,7 +2672,7 @@ int sqlite3WalEndWriteTransaction(Wal *pWal){ */ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ int rc = SQLITE_OK; - if( ALWAYS(pWal->writeLock) ){ + if( pWal->writeLock ){ Pgno iMax = pWal->hdr.mxFrame; Pgno iFrame; @@ -2603,7 +2711,7 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ ** point in the event of a savepoint rollback (via WalSavepointUndo()). */ void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){ - assert( pWal->writeLock ); + /* assert( pWal->writeLock ); */ aWalData[0] = pWal->hdr.mxFrame; aWalData[1] = pWal->hdr.aFrameCksum[0]; aWalData[2] = pWal->hdr.aFrameCksum[1]; @@ -2619,7 +2727,7 @@ void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){ int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ int rc = SQLITE_OK; - assert( pWal->writeLock ); + assert( pWal->writeLock || aWalData[0]==pWal->hdr.mxFrame ); assert( aWalData[3]!=pWal->nCkpt || aWalData[0]<=pWal->hdr.mxFrame ); if( aWalData[3]!=pWal->nCkpt ){ @@ -2783,6 +2891,7 @@ int sqlite3WalFrames( int szFrame; /* The size of a single frame */ i64 iOffset; /* Next byte to write in WAL file */ WalWriter w; /* The writer */ + int bUpgrade = 0; /* True if commit requires snapshot upgrade */ assert( pList ); assert( pWal->writeLock ); @@ -2798,6 +2907,22 @@ int sqlite3WalFrames( } #endif + if( isCommit ){ + volatile WalIndexHdr *pHead = walIndexHdr(pWal); + if( pHead->mxFrame>pWal->hdr.mxFrame ){ + if( memcmp((void*)&pHead[0], (void*)&pHead[1], sizeof(WalIndexHdr))!=0 ){ + /* TODO: Deal with this case. It's quite possible, but fiddly. */ + return SQLITE_CORRUPT_BKPT; + } + memcpy(&pWal->hdr, (void*)pHead, sizeof(WalIndexHdr)); + if( nTruncatehdr.nPage ){ + /* Do not truncate the database file in this case */ + nTruncate = pWal->hdr.nPage; + } + bUpgrade = 1; + } + } + /* See if it is possible to write these frames into the start of the ** log file, instead of appending to it at pWal->hdr.mxFrame. */ @@ -2946,6 +3071,14 @@ int sqlite3WalFrames( } } + if( rc==SQLITE_OK && bUpgrade ){ + /* If this commit required a snapshot upgrade, the pager cache is + ** not currently consistent with the head of the wal file. Zeroing + ** Wal.hdr here forces the next transaction to reset the cache + ** before beginning to read the db. */ + memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); + } + WALTRACE(("WAL%p: frame write %s\n", pWal, rc ? "failed" : "ok")); return rc; } @@ -3147,6 +3280,13 @@ int sqlite3WalHeapMemory(Wal *pWal){ return (pWal && pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ); } +/* +** Return true if in a write transaction, false otherwise. +*/ +int sqlite3WalIsInTrans(Wal *pWal){ + return (int)pWal->writeLock; +} + #ifdef SQLITE_ENABLE_ZIPVFS /* ** If the argument is not NULL, it points to a Wal object that holds a diff --git a/src/wal.h b/src/wal.h index 092546354b..db341caa04 100644 --- a/src/wal.h +++ b/src/wal.h @@ -126,6 +126,11 @@ int sqlite3WalExclusiveMode(Wal *pWal, int op); */ int sqlite3WalHeapMemory(Wal *pWal); +/* Return true if the WRITER lock is held. False otherwise. */ +int sqlite3WalIsInTrans(Wal *pWal); +int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pDirtyList, PgHdr *pPage1); +int sqlite3WalCommitRequiresUpgrade(Wal *pWal); + #ifdef SQLITE_ENABLE_ZIPVFS /* If the WAL file is not empty, return the number of bytes of content ** stored in each frame (i.e. the db page-size when the WAL was created). diff --git a/test/unlocked.test b/test/unlocked.test new file mode 100644 index 0000000000..e5f7868a20 --- /dev/null +++ b/test/unlocked.test @@ -0,0 +1,307 @@ +# 2015 July 26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set ::testprefix unlocked + + +do_execsql_test 1.0 { + PRAGMA journal_mode = wal; +} {wal} + +do_execsql_test 1.1 { + CREATE TABLE t1(k INTEGER PRIMARY KEY, v); + BEGIN UNLOCKED; + INSERT INTO t1 VALUES(1, 'abcd'); + COMMIT; +} + +do_execsql_test 1.2 { + SELECT * FROM t1; +} {1 abcd} + +do_execsql_test 1.3 { + BEGIN UNLOCKED; + INSERT INTO t1 VALUES(2, 'efgh'); + ROLLBACK; +} + +do_execsql_test 1.4 { + SELECT * FROM t1; +} {1 abcd} + + +#------------------------------------------------------------------------- +# UNLOCKED transactions cannot do cache spills. +# +foreach {tn trans spill} { + 1 {BEGIN UNLOCKED} 0 + 2 {BEGIN} 1 +} { + do_test 1.5.$tn { + sqlite3 db2 test.db + set walsz [file size test.db-wal] + + execsql { PRAGMA cache_size = 10 } db2 + execsql $trans db2 + execsql { + WITH cnt(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM cnt WHERE i<50) + INSERT INTO t1(v) SELECT randomblob(900) FROM cnt; + } db2 + + expr {[file size test.db-wal]==$walsz} + } [expr !$spill] + + execsql ROLLBACK db2 + db2 close +} + +#------------------------------------------------------------------------- +# UNLOCKED transactions man not be committed while there are active +# readers. +do_execsql_test 1.6.setup { + DROP TABLE t1; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); +} +foreach {tn trans commit_ok} { + 1 {BEGIN UNLOCKED} 0 + 2 {BEGIN} 1 +} { + do_test 1.6.$tn.1 { + set stmt [sqlite3_prepare db "SELECT * FROM t1" -1 dummy] + sqlite3_step $stmt + } SQLITE_ROW + do_test 1.6.$tn.2 { + execsql $trans + execsql { INSERT INTO t1 VALUES(7, 8) } + } {} + + if { $commit_ok } { + do_test 1.6.$tn.3 { catchsql COMMIT } {0 {}} + } else { + do_test 1.6.$tn.4 { catchsql COMMIT } {/1 {cannot commit transaction .*}/} + } + + sqlite3_finalize $stmt + catchsql ROLLBACK +} + +#------------------------------------------------------------------------- +# UNLOCKED transactions may not modify the db schema. +# +foreach {tn sql} { + 1 { CREATE TABLE xx(a, b) } + 2 { DROP TABLE t1 } +} { + do_catchsql_test 1.7.$tn.1 " + BEGIN UNLOCKED; + $sql + " {1 {cannot modify database schema - UNLOCKED transaction}} + + do_execsql_test 1.7.$tn.2 ROLLBACK +} + + +do_multiclient_test tn { + + #----------------------------------------------------------------------- + # 1. Start an UNLOCKED transaction using [db1]. + # + # 2. Start and then rollback a regular transaction using [db2]. This + # can be done as the ongoing [db1] transaction is UNLOCKED. + # + # 3. The [db1] transaction can now be committed, as [db2] has relinquished + # the write lock. + # + do_test 2.$tn.1.1 { + sql1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(k INTEGER PRIMARY KEY, v); + INSERT INTO t1 VALUES(1, 'one'); + } + sql1 { + BEGIN UNLOCKED; + INSERT INTO t1 VALUES(2, 'two'); + } + code1 { sqlite3_get_autocommit db } + } 0 + + do_test 2.$tn.1.2 { + sql2 { + BEGIN; + INSERT INTO t1 VALUES(3, 'three'); + ROLLBACK; + } + } {} + + do_test 2.$tn.1.3 { + sql1 COMMIT + sql2 { SELECT * FROM t1 } + } {1 one 2 two} + + #----------------------------------------------------------------------- + # 1. Start an UNLOCKED transaction using [db1]. + # + # 2. Commit a transaction using [db2]. + # + # 3. Try to commit with [db1]. Check that SQLITE_BUSY_SNAPSHOT is returned, + # and the transaction is not rolled back. + # + do_test 2.$tn.2.1 { + sql1 { + BEGIN UNLOCKED; + INSERT INTO t1 VALUES(-1, 'hello world'); + } + } {} + + do_test 2.$tn.2.2 { + sql2 { + INSERT INTO t1 VALUES(3, 'three'); + } + } {} + + do_test 2.$tn.2.3.1 { + set rc [catch { sql1 COMMIT } msg] + list $rc $msg + } {1 {database is locked}} + + do_test 2.$tn.2.3.2 { + code1 { list [sqlite3_extended_errcode db] [sqlite3_get_autocommit db] } + } {SQLITE_BUSY_SNAPSHOT 0} + + do_test 2.$tn.2.3.3 { + sql1 { + SELECT * FROM t1; + ROLLBACK; + } + } {-1 {hello world} 1 one 2 two} + + #----------------------------------------------------------------------- + # 1. Start an UNLOCKED transaction using [db1]. + # + # 2. Open a transaction using [db2]. + # + # 3. Try to commit with [db1]. Check that SQLITE_BUSY is returned, + # and the transaction is not rolled back. + # + # 4. Have [db2] roll its transaction back. Then check that [db1] can + # commit. + # + do_test 2.$tn.3.1 { + sql1 { + BEGIN UNLOCKED; + INSERT INTO t1 VALUES(4, 'four'); + } + } {} + + do_test 2.$tn.3.2 { + sql2 { + BEGIN; + INSERT INTO t1 VALUES(-1, 'xyz'); + } + } {} + + do_test 2.$tn.3.3.1 { + set rc [catch { sql1 COMMIT } msg] + list $rc $msg + } {1 {database is locked}} + + do_test 2.$tn.3.3.2 { + code1 { list [sqlite3_extended_errcode db] [sqlite3_get_autocommit db] } + } {SQLITE_BUSY 0} + + do_test 2.$tn.3.3.3 { + sql1 { SELECT * FROM t1; } + } {1 one 2 two 3 three 4 four} + + do_test 2.$tn.3.4 { + sql2 ROLLBACK + sql1 COMMIT + sql1 { SELECT * FROM t1; } + } {1 one 2 two 3 three 4 four} + + #----------------------------------------------------------------------- + # 1. Create a second table - t2. + # + # 2. Write to t1 with [db] and t2 with [db2]. + # + # 3. See if it worked. + # + do_test 2.$tn.4.1 { + sql1 { CREATE TABLE t2(a, b) } + } {} + do_test 2.$tn.4.2 { + sql2 { + BEGIN UNLOCKED; + INSERT INTO t2 VALUES('i', 'n'); + } + + sql1 { + BEGIN UNLOCKED; + INSERT INTO t1 VALUES(5, 'five'); + COMMIT; + } + + sql2 COMMIT + } {} + + do_test 2.$tn.4.3.1 { + sql2 {SELECT * FROM t1} + } {1 one 2 two 3 three 4 four 5 five} + do_test 2.$tn.4.3.2 { + sql1 {SELECT * FROM t1} + } {1 one 2 two 3 three 4 four 5 five} + + do_test 2.$tn.4.3.3 { sql2 {SELECT * FROM t2} } {i n} + do_test 2.$tn.4.3.4 { sql1 {SELECT * FROM t2} } {i n} + + #----------------------------------------------------------------------- + # The "schema cookie" issue. + # + # 1. Begin and UNLOCKED write to "t1" using [db] + # + # 2. Create an index on t1 using [db2]. + # + # 3. Attempt to commit the UNLOCKED write. This is an SQLITE_BUSY_SNAPSHOT, + # even though there is no page collision. + # + + do_test 2.$tn.5.1 { + sql1 { + BEGIN UNLOCKED; + INSERT INTO t1 VALUES(6, 'six'); + } + } {} + + do_test 2.$tn.5.2 { + sql2 { CREATE INDEX i1 ON t1(v); } + } {} + + do_test 2.$tn.5.3 { + list [catch { sql1 { COMMIT } } msg] $msg [sqlite3_errcode db] + } {1 {database is locked} SQLITE_BUSY_SNAPSHOT} + + do_test 2.$tn.5.4 { + sql2 { PRAGMA integrity_check } + } {ok} + catch { sql1 ROLLBACK } + +} + + + +finish_test diff --git a/tool/mkkeywordhash.c b/tool/mkkeywordhash.c index 721611f5a3..d1810cbeb9 100644 --- a/tool/mkkeywordhash.c +++ b/tool/mkkeywordhash.c @@ -262,6 +262,7 @@ static Keyword aKeywordTable[] = { { "TRIGGER", "TK_TRIGGER", TRIGGER }, { "UNION", "TK_UNION", COMPOUND }, { "UNIQUE", "TK_UNIQUE", ALWAYS }, + { "UNLOCKED", "TK_UNLOCKED", ALWAYS }, { "UPDATE", "TK_UPDATE", ALWAYS }, { "USING", "TK_USING", ALWAYS }, { "VACUUM", "TK_VACUUM", VACUUM }, From 37d36205f3e7061e5d17e1ed6c291bbcc0875bae Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 28 Jul 2015 16:46:49 +0000 Subject: [PATCH 002/179] Add some test cases and fix some small problems with BEGIN UNLOCKED transactions. FossilOrigin-Name: 6da0e962ad2aa5e52c1f1b5c3dbf77a2cb16ac2d --- manifest | 17 ++--- manifest.uuid | 2 +- src/wal.c | 181 ++++++++++++++++++++++++--------------------- test/unlocked.test | 66 ++++++++++++++++- 4 files changed, 170 insertions(+), 96 deletions(-) diff --git a/manifest b/manifest index 62addb80e6..c4c044ae5c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sexperimental\s"BEGIN\sUNLOCKED"\scommand. -D 2015-07-27T19:31:45.102 +C Add\ssome\stest\scases\sand\sfix\ssome\ssmall\sproblems\swith\sBEGIN\sUNLOCKED\stransactions. +D 2015-07-28T16:46:49.291 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 4de3ef40c8b3b75c0c55ff4242a43c8ce1ad90ee F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -402,7 +402,7 @@ F src/vdbesort.c f5009e7a35e3065635d8918b9a31f498a499976b F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 F src/vtab.c 082b35a25a26e3d36f365ca8cd73c1922532f05e F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb -F src/wal.c d588e26ed0d912584551dfe65d644d8c27cc2d1f +F src/wal.c a2e35f042103519ce22050acaf167960446963c7 F src/wal.h 5188cd85398dc612ae5ed2ed02d193377218be56 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c 909eba3b6db984eb2adfbca9de2c237ee7056adb @@ -1215,7 +1215,7 @@ F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unique2.test 41e7f83c6827605991160a31380148a9fc5f1339 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 -F test/unlocked.test edd4ed073ab16c0f8c9f5124e039100657b422c9 +F test/unlocked.test 854f3f428bb2e0ae91cb2400596a8f1ab7eaa409 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b @@ -1367,10 +1367,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b8e92227a469de677a66da62e4361f099c0b79d0 -R 889966cd4fab23d4f4355c880be4edd8 -T *branch * experimental-begin-unlocked -T *sym-experimental-begin-unlocked * -T -sym-trunk * +P 8079421604dbd40d03471dad6d12115119b554c2 +R 52e6230fe0fe2d8cb4434e54d864e964 U dan -Z 55f5e7a6a42bdc95bbb980b319cd8658 +Z 849da1cda4069d7075ecc7190a9058c5 diff --git a/manifest.uuid b/manifest.uuid index 25c9006198..23abb2a199 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8079421604dbd40d03471dad6d12115119b554c2 \ No newline at end of file +6da0e962ad2aa5e52c1f1b5c3dbf77a2cb16ac2d \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index a2200e7005..be018ebc99 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2508,6 +2508,34 @@ Pgno sqlite3WalDbsize(Wal *pWal){ return 0; } +/* +** Take the WRITER lock on the WAL file. Return SQLITE_OK if successful, +** or an SQLite error code otherwise. This routine does not invoke any +** busy-handler callbacks, that is done at a higher level. +*/ +static int walWriteLock(Wal *pWal){ + int rc; + + /* Cannot start a write transaction without first holding a read lock */ + assert( pWal->readLock>=0 ); + assert( pWal->writeLock==0 ); + + /* If this is a read-only connection, obtaining a write-lock is not + ** possible. In this case return SQLITE_READONLY. Otherwise, attempt + ** to grab the WRITER lock. Set Wal.writeLock to true and return + ** SQLITE_OK if successful, or leave Wal.writeLock clear and return + ** an SQLite error code (possibly SQLITE_BUSY) otherwise. */ + if( pWal->readOnly ){ + rc = SQLITE_READONLY; + }else{ + rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1, 0); + if( rc==SQLITE_OK ){ + pWal->writeLock = 1; + } + } + + return rc; +} /* ** This function starts a write transaction on the WAL. @@ -2523,35 +2551,18 @@ Pgno sqlite3WalDbsize(Wal *pWal){ ** There can only be a single writer active at a time. */ int sqlite3WalBeginWriteTransaction(Wal *pWal){ - int rc; - - /* Cannot start a write transaction without first holding a read - ** transaction. */ - assert( pWal->readLock>=0 ); - - if( pWal->readOnly ){ - return SQLITE_READONLY; - } - - /* Only one writer allowed at a time. Get the write lock. Return - ** SQLITE_BUSY if unable. - */ - rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1, 0); - if( rc ){ - return rc; - } - pWal->writeLock = 1; - - /* If another connection has written to the database file since the - ** time the read transaction on this connection was started, then - ** the write is disallowed. - */ - if( memcmp(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0 ){ - walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); - pWal->writeLock = 0; - rc = SQLITE_BUSY_SNAPSHOT; + int rc = walWriteLock(pWal); + if( rc==SQLITE_OK ){ + /* If another connection has written to the database file since the + ** time the read transaction on this connection was started, then + ** the write is disallowed. Release the WRITER lock and return + ** SQLITE_BUSY_SNAPSHOT in this case. */ + if( memcmp(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0 ){ + walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); + pWal->writeLock = 0; + rc = SQLITE_BUSY_SNAPSHOT; + } } - return rc; } @@ -2563,25 +2574,7 @@ int sqlite3WalBeginWriteTransaction(Wal *pWal){ ** the wal file. */ int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pList, PgHdr *pPage1){ - volatile WalIndexHdr *pHead; /* Head of the wal file */ - int rc; - - /* Cannot start a write transaction without first holding a read - ** transaction. */ - assert( pWal->readLock>=0 ); - - if( pWal->readOnly ){ - return SQLITE_READONLY; - } - - /* Only one writer allowed at a time. Get the write lock. Return - ** SQLITE_BUSY if unable. - */ - rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1, 0); - if( rc ){ - return rc; - } - pWal->writeLock = 1; + int rc = walWriteLock(pWal); /* If the database has been modified since this transaction was started, ** check if it is still possible to commit. The transaction can be @@ -2593,41 +2586,53 @@ int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pList, PgHdr *pPage1){ ** b) The database schema cookie has not been modified since the ** transaction was started. */ - pHead = walIndexHdr(pWal); - if( memcmp(&pWal->hdr, (void*)pHead, sizeof(WalIndexHdr))!=0 ){ - /* TODO: Is this safe? Because it holds the WRITER lock this thread - ** has exclusive access to the live header, but might it be corrupt? */ - PgHdr *pPg; - u32 iLast = pHead->mxFrame; - for(pPg=pList; rc==SQLITE_OK && pPg; pPg=pPg->pDirty){ - u32 iSlot = 0; - rc = walFindFrame(pWal, pPg->pgno, iLast, &iSlot); - if( iSlot>pWal->hdr.mxFrame ){ - sqlite3_log(SQLITE_OK, - "cannot commit UNLOCKED transaction (conflict at page %d)", - (int)pPg->pgno - ); - rc = SQLITE_BUSY_SNAPSHOT; - } - } - - if( rc==SQLITE_OK ){ - /* Read the newest schema cookie from the wal file. */ - u32 iSlot = 0; - rc = walFindFrame(pWal, 1, iLast, &iSlot); - if( rc==SQLITE_OK && iSlot>pWal->hdr.mxFrame ){ - u8 aNew[4]; - u8 *aOld = &((u8*)pPage1->pData)[40]; - int sz; - i64 iOffset; - sz = pWal->hdr.szPage; - sz = (sz&0xfe00) + ((sz&0x0001)<<16); - iOffset = walFrameOffset(iSlot, sz) + WAL_FRAME_HDRSIZE + 40; - rc = sqlite3OsRead(pWal->pWalFd, aNew, sizeof(aNew), iOffset); - if( rc==SQLITE_OK && memcmp(aOld, aNew, sizeof(aNew)) ){ - /* TODO: New error code? SQLITE_BUSY_SCHEMA. */ + if( rc==SQLITE_OK ){ + volatile WalIndexHdr *pHead; /* Head of the wal file */ + pHead = walIndexHdr(pWal); + if( memcmp(&pWal->hdr, (void*)pHead, sizeof(WalIndexHdr))!=0 ){ + int bSeenPage1 = 0; /* True if page 1 is in list pList */ + + /* TODO: Is this safe? Because it holds the WRITER lock this thread + ** has exclusive access to the live header, but might it be corrupt? */ + PgHdr *pPg; + u32 iLast = pHead->mxFrame; + for(pPg=pList; rc==SQLITE_OK && pPg; pPg=pPg->pDirty){ + u32 iSlot = 0; + rc = walFindFrame(pWal, pPg->pgno, iLast, &iSlot); + if( iSlot>pWal->hdr.mxFrame ){ + sqlite3_log(SQLITE_OK, + "cannot commit UNLOCKED transaction (conflict at page %d)", + (int)pPg->pgno + ); rc = SQLITE_BUSY_SNAPSHOT; } + if( pPg->pgno==1 ) bSeenPage1 = 1; + } + + /* If the current transaction does not modify page 1 of the database, + ** check if page 1 has been modified since the transaction was started. + ** If it has, check if the schema cookie value (the 4 bytes beginning at + ** byte offset 40) is the same as it is on pPage1. If not, this indicates + ** that the current head of the wal file uses a different schema than + ** the snapshot against which the current transaction was prepared. Return + ** SQLITE_BUSY_SNAPSHOT in this case. */ + if( rc==SQLITE_OK && bSeenPage1==0 ){ + u32 iSlot = 0; + rc = walFindFrame(pWal, 1, iLast, &iSlot); + if( rc==SQLITE_OK && iSlot>pWal->hdr.mxFrame ){ + u8 aNew[4]; + u8 *aOld = &((u8*)pPage1->pData)[40]; + int sz; + i64 iOffset; + sz = pWal->hdr.szPage; + sz = (sz&0xfe00) + ((sz&0x0001)<<16); + iOffset = walFrameOffset(iSlot, sz) + WAL_FRAME_HDRSIZE + 40; + rc = sqlite3OsRead(pWal->pWalFd, aNew, sizeof(aNew), iOffset); + if( rc==SQLITE_OK && memcmp(aOld, aNew, sizeof(aNew)) ){ + /* TODO: New error code? SQLITE_BUSY_SCHEMA. */ + rc = SQLITE_BUSY_SNAPSHOT; + } + } } } } @@ -2636,9 +2641,18 @@ int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pList, PgHdr *pPage1){ } /* -** The caller holds the WRITER lock. This function returns true if a snapshot -** upgrade is required before the transaction can be committed, or false -** otherwise. +** This function is only ever called while committing an UNLOCKED +** transaction, after the caller has already obtained the WRITER lock +** (by calling the sqlite3WalLockForCommit() routine). This function +** returns true if the transaction was prepared against a database +** snapshot older than the current head of the wal file. +** +** Note that this will only work as described if the database is +** currently executing an UNLOCKED transaction, as it assumes that +** pWal->hdr has not been modified since the beginning of the +** transaction. This may not be true for a non-UNLOCKED transaction, +** as pWal->hdr is updated if any pages are spilled to the wal file +** while the transaction is executing. */ int sqlite3WalCommitRequiresUpgrade(Wal *pWal){ assert( pWal->writeLock ); @@ -2711,7 +2725,6 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ ** point in the event of a savepoint rollback (via WalSavepointUndo()). */ void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){ - /* assert( pWal->writeLock ); */ aWalData[0] = pWal->hdr.mxFrame; aWalData[1] = pWal->hdr.aFrameCksum[0]; aWalData[2] = pWal->hdr.aFrameCksum[1]; diff --git a/test/unlocked.test b/test/unlocked.test index e5f7868a20..c7ae0466e5 100644 --- a/test/unlocked.test +++ b/test/unlocked.test @@ -279,7 +279,6 @@ do_multiclient_test tn { # 3. Attempt to commit the UNLOCKED write. This is an SQLITE_BUSY_SNAPSHOT, # even though there is no page collision. # - do_test 2.$tn.5.1 { sql1 { BEGIN UNLOCKED; @@ -300,6 +299,71 @@ do_multiclient_test tn { } {ok} catch { sql1 ROLLBACK } + #----------------------------------------------------------------------- + # The "schema cookie" issue. + # + # 1. Begin an UNLOCKED write to "t1" using [db] + # + # 2. Lots of inserts into t2. Enough to grow the db file. + # + # 3. Check that the UNLOCKED transaction can still be committed. + # + do_test 2.$tn.6.1 { + sql1 { + BEGIN UNLOCKED; + INSERT INTO t1 VALUES(6, 'six'); + } + } {} + + do_test 2.$tn.6.2 { + sql2 { + WITH src(a,b) AS ( + VALUES(1,1) UNION ALL SELECT a+1,b+1 FROM src WHERE a<10000 + ) INSERT INTO t2 SELECT * FROM src; + } + } {} + + do_test 2.$tn.6.3 { + sql1 { + SELECT count(*) FROM t2; + COMMIT; + SELECT count(*) FROM t2; + } + } {1 10001} + + #----------------------------------------------------------------------- + # + # 1. Begin an big UNLOCKED write to "t1" using [db] - large enough to + # grow the db file. + # + # 2. Lots of inserts into t2. Also enough to grow the db file. + # + # 3. Check that the UNLOCKED transaction cannot be committed (due to a clash + # on page 1 - the db size field). + # + do_test 2.$tn.7.1 { + sql1 { + BEGIN UNLOCKED; + WITH src(a,b) AS ( + VALUES(10000,10000) UNION ALL SELECT a+1,b+1 FROM src WHERE a<20000 + ) INSERT INTO t1 SELECT * FROM src; + } + } {} + + do_test 2.$tn.7.2 { + sql2 { + WITH src(a,b) AS ( + VALUES(1,1) UNION ALL SELECT a+1,b+1 FROM src WHERE a<10000 + ) INSERT INTO t2 SELECT * FROM src; + } + } {} + + do_test 2.$tn.7.3 { + list [catch { sql1 { COMMIT } } msg] $msg [sqlite3_errcode db] + } {1 {database is locked} SQLITE_BUSY_SNAPSHOT} + sql1 ROLLBACK + + } From 773d2d6c097980a4b4c9c0a12b0e0c57f893b330 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 29 Jul 2015 12:14:28 +0000 Subject: [PATCH 003/179] Only allow UNLOCKED transactions to commit if none of the pages read by the transaction have been modified since it was opened. FossilOrigin-Name: 0b9718426e44df092850c5d095ce1b84a1e567cf --- manifest | 26 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 12 -------- src/btree.h | 2 -- src/pager.c | 50 +++++++++++++++++++++---------- src/pager.h | 2 +- src/vdbeaux.c | 4 ++- src/wal.c | 75 +++++++++++++++++----------------------------- src/wal.h | 3 +- test/unlocked.test | 16 ++++++---- 10 files changed, 90 insertions(+), 102 deletions(-) diff --git a/manifest b/manifest index c4c044ae5c..2f36ddd5b8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssome\stest\scases\sand\sfix\ssome\ssmall\sproblems\swith\sBEGIN\sUNLOCKED\stransactions. -D 2015-07-28T16:46:49.291 +C Only\sallow\sUNLOCKED\stransactions\sto\scommit\sif\snone\sof\sthe\spages\sread\sby\sthe\stransaction\shave\sbeen\smodified\ssince\sit\swas\sopened. +D 2015-07-29T12:14:28.276 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 4de3ef40c8b3b75c0c55ff4242a43c8ce1ad90ee F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -270,8 +270,8 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 4d9134dc988a87838c06056c89c0e8c4700a0452 F src/bitvec.c d1f21d7d91690747881f03940584f4cc548c9d3d F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 -F src/btree.c b184ce6d483dfadc53126a4b7994633ec0d5ef5f -F src/btree.h 550e8447684013a69196ec664142f67538da8ef1 +F src/btree.c 5087b0a1358abd1449165dd14d49962bc3ad9f44 +F src/btree.h 40bd41ef0b71d6f7502725dc159fa0d6bebd8bb3 F src/btreeInt.h 2ad754dd4528baa8d0946a593cc373b890bf859e F src/build.c 28c15c43eefc0066ff64040526ff649c32fe5523 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 @@ -315,8 +315,8 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 388c023582b17890f10c980b30ec1922b471753b F src/os_win.c 40b3af7a47eb1107d0d69e592bec345a3b7b798a F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c c9f22eb831851220c1095863c5797a88f71bea8a -F src/pager.h b6cf6e9552c5b1e7d6d0d8cd7ca51be7802bec72 +F src/pager.c 18aa67541bd2c7d7963f180669ea4e48cd5246f4 +F src/pager.h 41938212a5d80a7068883bb29ea6ac0d4ef3a6d8 F src/parse.y 199145cd982587d70fa3db2f9b11f7ab118f0acd F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0 F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9 @@ -395,15 +395,15 @@ F src/vdbe.c fd6832408d9621ad347a65b01a05380fe766232f F src/vdbe.h 7a75045d879118b9d3af7e8b3c108f2f27c51473 F src/vdbeInt.h 8b54e01ad0463590e7cffabce0bc36da9ee4f816 F src/vdbeapi.c adabbd66eb2e3a10f3998485ee0be7e326d06ee4 -F src/vdbeaux.c 27feae106f3f4db298e65d37a9da4962aaff724b +F src/vdbeaux.c 24a03ccbadc8d4cfecf25b6399db95851b10dd21 F src/vdbeblob.c 4f2e8e075d238392df98c5e03a64342465b03f90 F src/vdbemem.c ae38a0d35ae71cf604381a887c170466ba518090 F src/vdbesort.c f5009e7a35e3065635d8918b9a31f498a499976b F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 F src/vtab.c 082b35a25a26e3d36f365ca8cd73c1922532f05e F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb -F src/wal.c a2e35f042103519ce22050acaf167960446963c7 -F src/wal.h 5188cd85398dc612ae5ed2ed02d193377218be56 +F src/wal.c 4497b466fee8ed56c31672bb35741dd0321ec999 +F src/wal.h cd7ea52439d56571d549b7d4eacdeed58d841385 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c 909eba3b6db984eb2adfbca9de2c237ee7056adb F src/whereInt.h 5f87e3c4b0551747d119730dfebddd3c54f04047 @@ -1215,7 +1215,7 @@ F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unique2.test 41e7f83c6827605991160a31380148a9fc5f1339 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 -F test/unlocked.test 854f3f428bb2e0ae91cb2400596a8f1ab7eaa409 +F test/unlocked.test 2da6645c3532985715d17dfcb240ef5f09c2e9d4 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b @@ -1367,7 +1367,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8079421604dbd40d03471dad6d12115119b554c2 -R 52e6230fe0fe2d8cb4434e54d864e964 +P 6da0e962ad2aa5e52c1f1b5c3dbf77a2cb16ac2d +R 74dcd93afc2d7e4cc222b666276012ce U dan -Z 849da1cda4069d7075ecc7190a9058c5 +Z 3bd2cb4e2cf9d30cb919b0a0d11717fa diff --git a/manifest.uuid b/manifest.uuid index 23abb2a199..f6215ad409 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6da0e962ad2aa5e52c1f1b5c3dbf77a2cb16ac2d \ No newline at end of file +0b9718426e44df092850c5d095ce1b84a1e567cf \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index c83a01ad4c..69d4873d0d 100644 --- a/src/btree.c +++ b/src/btree.c @@ -9590,15 +9590,3 @@ int sqlite3BtreeIsReadonly(Btree *p){ ** Return the size of the header added to each page by this module. */ int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); } - -int sqlite3BtreeExclusiveLock(Btree *p){ - int rc; - BtShared *pBt = p->pBt; - sqlite3BtreeEnter(p); - rc = sqlite3PagerExclusiveLock(pBt->pPager, pBt->pPage1->pDbPage); - sqlite3BtreeLeave(p); - return rc; -} - - - diff --git a/src/btree.h b/src/btree.h index ae9b2c4e59..50dc8ca290 100644 --- a/src/btree.h +++ b/src/btree.h @@ -270,6 +270,4 @@ void sqlite3BtreeCursorList(Btree*); # define sqlite3SchemaMutexHeld(X,Y,Z) 1 #endif -int sqlite3BtreeExclusiveLock(Btree*); - #endif /* _BTREE_H_ */ diff --git a/src/pager.c b/src/pager.c index 2ea3b90a96..5107edda36 100644 --- a/src/pager.c +++ b/src/pager.c @@ -657,6 +657,7 @@ struct Pager { u32 cksumInit; /* Quasi-random value added to every checksum */ u32 nSubRec; /* Number of records written to sub-journal */ Bitvec *pInJournal; /* One bit for each page in the database file */ + Bitvec *pAllRead; /* Pages read within current UNLOCKED trans. */ sqlite3_file *fd; /* File descriptor for database */ sqlite3_file *jfd; /* File descriptor for main journal */ sqlite3_file *sjfd; /* File descriptor for sub-journal */ @@ -1738,6 +1739,16 @@ static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ return rc; } +/* +** Free the Pager.pInJournal and Pager.pAllRead bitvec objects. +*/ +static void pagerFreeBitvecs(Pager *pPager){ + sqlite3BitvecDestroy(pPager->pInJournal); + sqlite3BitvecDestroy(pPager->pAllRead); + pPager->pInJournal = 0; + pPager->pAllRead = 0; +} + /* ** This function is a no-op if the pager is in exclusive mode and not ** in the ERROR state. Otherwise, it switches the pager to PAGER_OPEN @@ -1762,8 +1773,7 @@ static void pager_unlock(Pager *pPager){ || pPager->eState==PAGER_ERROR ); - sqlite3BitvecDestroy(pPager->pInJournal); - pPager->pInJournal = 0; + pagerFreeBitvecs(pPager); releaseAllSavepoints(pPager); if( pagerUseWal(pPager) ){ @@ -1999,8 +2009,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ } #endif - sqlite3BitvecDestroy(pPager->pInJournal); - pPager->pInJournal = 0; + pagerFreeBitvecs(pPager); pPager->nRec = 0; sqlite3PcacheCleanAll(pPager->pPCache); sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); @@ -4082,7 +4091,7 @@ static int syncJournal(Pager *pPager, int newHdr){ assert( assert_pager_state(pPager) ); assert( !pagerUseWal(pPager) ); - rc = sqlite3PagerExclusiveLock(pPager, 0); + rc = sqlite3PagerExclusiveLock(pPager); if( rc!=SQLITE_OK ) return rc; if( !pPager->noSync ){ @@ -4432,7 +4441,7 @@ static int pagerStress(void *p, PgHdr *pPg){ if( pagerUseWal(pPager) ){ /* If the transaction is a "BEGIN UNLOCKED" transaction, the page ** cannot be flushed to disk. Return early in this case. */ - if( sqlite3WalIsInTrans(pPager->pWal)==0 ) return SQLITE_OK; + if( pPager->pAllRead ) return SQLITE_OK; /* Write a single frame for this page to the log. */ rc = subjournalPageIfRequired(pPg); @@ -5274,6 +5283,14 @@ int sqlite3PagerAcquire( } pPager->hasBeenUsed = 1; + /* If this is an UNLOCKED transaction and the page being read was + ** present in the database file when the transaction was opened, + ** mark it as read in the pAllRead vector. */ + if( pPager->pAllRead && pgno<=pPager->dbOrigSize ){ + rc = sqlite3BitvecSet(pPager->pAllRead, pgno); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + } + /* If the pager is in the error state, return an error immediately. ** Otherwise, request the page from the PCache layer. */ if( pPager->errCode!=SQLITE_OK ){ @@ -5578,6 +5595,7 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ if( ALWAYS(pPager->eState==PAGER_READER) ){ assert( pPager->pInJournal==0 ); + assert( pPager->pAllRead==0 ); if( pagerUseWal(pPager) ){ /* If the pager is configured to use locking_mode=exclusive, and an @@ -5598,6 +5616,13 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ */ if( exFlag>=0 ){ rc = sqlite3WalBeginWriteTransaction(pPager->pWal); + }else{ + pPager->pAllRead = sqlite3BitvecCreate(pPager->dbSize); + if( pPager->pAllRead==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3BitvecSet(pPager->pAllRead, 1); + } } }else{ /* Obtain a RESERVED lock on the database file. If the exFlag parameter @@ -6066,7 +6091,7 @@ int sqlite3PagerSync(Pager *pPager, const char *zMaster){ ** Otherwise, either SQLITE_BUSY or an SQLITE_IOERR_XXX error code is ** returned. */ -int sqlite3PagerExclusiveLock(Pager *pPager, PgHdr *pPage1){ +int sqlite3PagerExclusiveLock(Pager *pPager){ int rc = SQLITE_OK; assert( pPager->eState==PAGER_WRITER_CACHEMOD || pPager->eState==PAGER_WRITER_DBMOD @@ -6076,21 +6101,14 @@ int sqlite3PagerExclusiveLock(Pager *pPager, PgHdr *pPage1){ if( 0==pagerUseWal(pPager) ){ rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); }else{ - Wal *pWal = pPager->pWal; - if( 0==sqlite3WalIsInTrans(pWal) ){ - /* TODO: There must be an optimization opportunity here, as this call - ** to PcacheDirtyList() sorts the list of dirty pages, even though it - ** is not really required - and will be sorted again in CommitPhaseOne() - ** in any case. */ - PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); - + if( pPager->pAllRead ){ /* This is an UNLOCKED transaction. Attempt to lock the wal database ** here. If SQLITE_BUSY (but not SQLITE_BUSY_SNAPSHOT) is returned, ** invoke the busy-handler and try again for as long as it returns ** non-zero. */ do { /* rc = sqlite3WalBeginWriteTransaction(pWal); */ - rc = sqlite3WalLockForCommit(pWal, pList, pPage1); + rc = sqlite3WalLockForCommit(pPager->pWal, pPager->pAllRead); }while( rc==SQLITE_BUSY && pPager->xBusyHandler(pPager->pBusyHandlerArg) ); diff --git a/src/pager.h b/src/pager.h index 57d2812701..3c0b9a1a71 100644 --- a/src/pager.h +++ b/src/pager.h @@ -150,7 +150,7 @@ void *sqlite3PagerGetExtra(DbPage *); void sqlite3PagerPagecount(Pager*, int*); int sqlite3PagerBegin(Pager*, int exFlag, int); int sqlite3PagerCommitPhaseOne(Pager*,const char *zMaster, int); -int sqlite3PagerExclusiveLock(Pager*, DbPage *pPage1); +int sqlite3PagerExclusiveLock(Pager*); int sqlite3PagerSync(Pager *pPager, const char *zMaster); int sqlite3PagerCommitPhaseTwo(Pager*); int sqlite3PagerRollback(Pager*); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index f5f9d0fbdd..93a7187ab9 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2020,7 +2020,9 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ if( sqlite3BtreeIsInTrans(pBt) ){ needXcommit = 1; if( i!=1 ) nTrans++; - rc = sqlite3BtreeExclusiveLock(pBt); + sqlite3BtreeEnter(pBt); + rc = sqlite3PagerExclusiveLock(sqlite3BtreePager(pBt)); + sqlite3BtreeLeave(pBt); } } diff --git a/src/wal.c b/src/wal.c index be018ebc99..5fbdae6a1a 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2573,7 +2573,7 @@ int sqlite3WalBeginWriteTransaction(Wal *pWal){ ** transaction. It may be assumed that no frames have been written to ** the wal file. */ -int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pList, PgHdr *pPage1){ +int sqlite3WalLockForCommit(Wal *pWal, Bitvec *pAllRead){ int rc = walWriteLock(pWal); /* If the database has been modified since this transaction was started, @@ -2590,49 +2590,35 @@ int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pList, PgHdr *pPage1){ volatile WalIndexHdr *pHead; /* Head of the wal file */ pHead = walIndexHdr(pWal); if( memcmp(&pWal->hdr, (void*)pHead, sizeof(WalIndexHdr))!=0 ){ - int bSeenPage1 = 0; /* True if page 1 is in list pList */ - /* TODO: Is this safe? Because it holds the WRITER lock this thread - ** has exclusive access to the live header, but might it be corrupt? */ - PgHdr *pPg; - u32 iLast = pHead->mxFrame; - for(pPg=pList; rc==SQLITE_OK && pPg; pPg=pPg->pDirty){ - u32 iSlot = 0; - rc = walFindFrame(pWal, pPg->pgno, iLast, &iSlot); - if( iSlot>pWal->hdr.mxFrame ){ - sqlite3_log(SQLITE_OK, - "cannot commit UNLOCKED transaction (conflict at page %d)", - (int)pPg->pgno - ); - rc = SQLITE_BUSY_SNAPSHOT; - } - if( pPg->pgno==1 ) bSeenPage1 = 1; - } - - /* If the current transaction does not modify page 1 of the database, - ** check if page 1 has been modified since the transaction was started. - ** If it has, check if the schema cookie value (the 4 bytes beginning at - ** byte offset 40) is the same as it is on pPage1. If not, this indicates - ** that the current head of the wal file uses a different schema than - ** the snapshot against which the current transaction was prepared. Return - ** SQLITE_BUSY_SNAPSHOT in this case. */ - if( rc==SQLITE_OK && bSeenPage1==0 ){ - u32 iSlot = 0; - rc = walFindFrame(pWal, 1, iLast, &iSlot); - if( rc==SQLITE_OK && iSlot>pWal->hdr.mxFrame ){ - u8 aNew[4]; - u8 *aOld = &((u8*)pPage1->pData)[40]; - int sz; - i64 iOffset; - sz = pWal->hdr.szPage; - sz = (sz&0xfe00) + ((sz&0x0001)<<16); - iOffset = walFrameOffset(iSlot, sz) + WAL_FRAME_HDRSIZE + 40; - rc = sqlite3OsRead(pWal->pWalFd, aNew, sizeof(aNew), iOffset); - if( rc==SQLITE_OK && memcmp(aOld, aNew, sizeof(aNew)) ){ - /* TODO: New error code? SQLITE_BUSY_SCHEMA. */ - rc = SQLITE_BUSY_SNAPSHOT; + ** has exclusive access to the live header, but might it be corrupt? + ** This code should check that the wal-index-header is Ok, and return + ** SQLITE_BUSY_SNAPSHOT if it is not. */ + int iHash; + int iLastHash = walFramePage(pHead->mxFrame); + for(iHash=walFramePage(pWal->hdr.mxFrame+1); iHash<=iLastHash; iHash++){ + volatile ht_slot *aHash; + volatile u32 *aPgno; + u32 iZero; + + rc = walHashGet(pWal, iHash, &aHash, &aPgno, &iZero); + if( rc==SQLITE_OK ){ + int i; + int iMin = (pWal->hdr.mxFrame+1 - iZero); + int iMax = (iHash==0) ? HASHTABLE_NPAGE_ONE : HASHTABLE_NPAGE; + if( iMin<1 ) iMin = 1; + if( iMax>pHead->mxFrame ) iMax = pHead->mxFrame; + for(i=iMin; i<=iMax; i++){ + if( sqlite3BitvecTestNotNull(pAllRead, aPgno[i]) ){ + sqlite3_log(SQLITE_OK, + "cannot commit UNLOCKED transaction (conflict at page %d)", + (int)aPgno[i] + ); + rc = SQLITE_BUSY_SNAPSHOT; + } } } + if( rc!=SQLITE_OK ) break; } } } @@ -3293,13 +3279,6 @@ int sqlite3WalHeapMemory(Wal *pWal){ return (pWal && pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ); } -/* -** Return true if in a write transaction, false otherwise. -*/ -int sqlite3WalIsInTrans(Wal *pWal){ - return (int)pWal->writeLock; -} - #ifdef SQLITE_ENABLE_ZIPVFS /* ** If the argument is not NULL, it points to a Wal object that holds a diff --git a/src/wal.h b/src/wal.h index db341caa04..fe85a52cc0 100644 --- a/src/wal.h +++ b/src/wal.h @@ -127,8 +127,7 @@ int sqlite3WalExclusiveMode(Wal *pWal, int op); int sqlite3WalHeapMemory(Wal *pWal); /* Return true if the WRITER lock is held. False otherwise. */ -int sqlite3WalIsInTrans(Wal *pWal); -int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pDirtyList, PgHdr *pPage1); +int sqlite3WalLockForCommit(Wal *pWal, Bitvec *pRead); int sqlite3WalCommitRequiresUpgrade(Wal *pWal); #ifdef SQLITE_ENABLE_ZIPVFS diff --git a/test/unlocked.test b/test/unlocked.test index c7ae0466e5..2a5862d3ac 100644 --- a/test/unlocked.test +++ b/test/unlocked.test @@ -300,13 +300,12 @@ do_multiclient_test tn { catch { sql1 ROLLBACK } #----------------------------------------------------------------------- - # The "schema cookie" issue. # # 1. Begin an UNLOCKED write to "t1" using [db] # - # 2. Lots of inserts into t2. Enough to grow the db file. + # 2. Lots of inserts into t2. Enough to grow the db file and modify page 1. # - # 3. Check that the UNLOCKED transaction can still be committed. + # 3. Check that the UNLOCKED transaction can not be committed. # do_test 2.$tn.6.1 { sql1 { @@ -324,12 +323,17 @@ do_multiclient_test tn { } {} do_test 2.$tn.6.3 { + sql1 { SELECT count(*) FROM t2 } + list [catch { sql1 { COMMIT } } msg] $msg [sqlite3_errcode db] + } {1 {database is locked} SQLITE_BUSY_SNAPSHOT} + sql1 ROLLBACK + + do_test 2.$tn.6.4 { sql1 { - SELECT count(*) FROM t2; - COMMIT; + SELECT count(*) FROM t1; SELECT count(*) FROM t2; } - } {1 10001} + } {5 10001} #----------------------------------------------------------------------- # From 1a9cde3ba9f2a9125886d26141686a4bca7f11de Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 15 Aug 2015 18:16:46 +0000 Subject: [PATCH 004/179] Handle writes to auto-vacuum databases within UNLOCKED transactions in the same way as for non-UNLOCKED transactions. FossilOrigin-Name: de1ea450db33b140b11af5b801ea6a15875e774e --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 2 +- test/unlocked.test | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 2f36ddd5b8..7ca1c00b94 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Only\sallow\sUNLOCKED\stransactions\sto\scommit\sif\snone\sof\sthe\spages\sread\sby\sthe\stransaction\shave\sbeen\smodified\ssince\sit\swas\sopened. -D 2015-07-29T12:14:28.276 +C Handle\swrites\sto\sauto-vacuum\sdatabases\swithin\sUNLOCKED\stransactions\sin\sthe\ssame\sway\sas\sfor\snon-UNLOCKED\stransactions. +D 2015-08-15T18:16:46.463 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 4de3ef40c8b3b75c0c55ff4242a43c8ce1ad90ee F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -270,7 +270,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 4d9134dc988a87838c06056c89c0e8c4700a0452 F src/bitvec.c d1f21d7d91690747881f03940584f4cc548c9d3d F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 -F src/btree.c 5087b0a1358abd1449165dd14d49962bc3ad9f44 +F src/btree.c c1bbc83539de1e749aa36ce9c283f95b50543bcb F src/btree.h 40bd41ef0b71d6f7502725dc159fa0d6bebd8bb3 F src/btreeInt.h 2ad754dd4528baa8d0946a593cc373b890bf859e F src/build.c 28c15c43eefc0066ff64040526ff649c32fe5523 @@ -1215,7 +1215,7 @@ F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unique2.test 41e7f83c6827605991160a31380148a9fc5f1339 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 -F test/unlocked.test 2da6645c3532985715d17dfcb240ef5f09c2e9d4 +F test/unlocked.test 644a84d7f032ca029d490f0b0bc8fb9f86c7a1a5 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b @@ -1367,7 +1367,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6da0e962ad2aa5e52c1f1b5c3dbf77a2cb16ac2d -R 74dcd93afc2d7e4cc222b666276012ce +P 0b9718426e44df092850c5d095ce1b84a1e567cf +R ac1d3857424ab06ed70d671704c3737d U dan -Z 3bd2cb4e2cf9d30cb919b0a0d11717fa +Z 12da9addf6141c212c3c7cdffe33d767 diff --git a/manifest.uuid b/manifest.uuid index f6215ad409..6a4d41072d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0b9718426e44df092850c5d095ce1b84a1e567cf \ No newline at end of file +de1ea450db33b140b11af5b801ea6a15875e774e \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 69d4873d0d..7f9054f309 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3142,8 +3142,8 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ if( (pBt->btsFlags & BTS_READ_ONLY)!=0 ){ rc = SQLITE_READONLY; }else{ + int exFlag = (p->db->bUnlocked && !ISAUTOVACUUM) ? -1 : (wrflag>1); int bSubjInMem = sqlite3TempInMemory(p->db); - int exFlag = p->db->bUnlocked ? -1 : (wrflag>1); assert( p->db->bUnlocked==0 || wrflag==1 ); rc = sqlite3PagerBegin(pBt->pPager, exFlag, bSubjInMem); if( rc==SQLITE_OK ){ diff --git a/test/unlocked.test b/test/unlocked.test index 2a5862d3ac..23eb5a5064 100644 --- a/test/unlocked.test +++ b/test/unlocked.test @@ -115,6 +115,41 @@ foreach {tn sql} { do_execsql_test 1.7.$tn.2 ROLLBACK } +#------------------------------------------------------------------------- +# If an auto-vacuum database is written within an UNLOCKED transaction, it +# is handled in the same way as for a non-UNLOCKED transaction. +# +reset_db +do_execsql_test 1.8.1 { + PRAGMA auto_vacuum = 1; + PRAGMA journal_mode = wal; + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES('x', 'y'); +} {wal} + +do_execsql_test 1.8.2 { + BEGIN UNLOCKED; + SELECT * FROM t1; + COMMIT; +} {x y} + +do_catchsql_test 1.8.3 { + BEGIN UNLOCKED; + INSERT INTO t1 VALUES('a', 'b'); +} {0 {}} + +do_test 1.8.4 { + sqlite3 db2 test.db + catchsql { + BEGIN UNLOCKED; + INSERT INTO t1 VALUES('c', 'd'); + } db2 +} {1 {database is locked}} + +do_test 1.8.5 { + db eval COMMIT + db2 eval COMMIT +} {} do_multiclient_test tn { From 7b3d71e9cc82ae2b7aad42234875b53b935b21af Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 19 Aug 2015 20:27:05 +0000 Subject: [PATCH 005/179] When committing an unlocked transaction, relocate newly allocated database pages within the file to avoid conflicting with committed transactions. There are lots of things still to fix in this code. FossilOrigin-Name: 3bbc31d515ba9fc920c5cbc7059d3eb1ba9c7f8e --- manifest | 29 ++-- manifest.uuid | 2 +- src/btree.c | 410 ++++++++++++++++++++++++++++++++++++++------ src/btree.h | 2 + src/btreeInt.h | 10 ++ src/pager.c | 37 ++-- src/pager.h | 10 +- src/vdbeaux.c | 4 +- src/wal.c | 59 ++++--- src/wal.h | 3 +- test/unlocked.test | 5 +- test/unlocked2.test | 101 +++++++++++ 12 files changed, 558 insertions(+), 114 deletions(-) create mode 100644 test/unlocked2.test diff --git a/manifest b/manifest index 7ca1c00b94..92f5317a3d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Handle\swrites\sto\sauto-vacuum\sdatabases\swithin\sUNLOCKED\stransactions\sin\sthe\ssame\sway\sas\sfor\snon-UNLOCKED\stransactions. -D 2015-08-15T18:16:46.463 +C When\scommitting\san\sunlocked\stransaction,\srelocate\snewly\sallocated\sdatabase\spages\swithin\sthe\sfile\sto\savoid\sconflicting\swith\scommitted\stransactions.\sThere\sare\slots\sof\sthings\sstill\sto\sfix\sin\sthis\scode. +D 2015-08-19T20:27:05.840 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 4de3ef40c8b3b75c0c55ff4242a43c8ce1ad90ee F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -270,9 +270,9 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 4d9134dc988a87838c06056c89c0e8c4700a0452 F src/bitvec.c d1f21d7d91690747881f03940584f4cc548c9d3d F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 -F src/btree.c c1bbc83539de1e749aa36ce9c283f95b50543bcb -F src/btree.h 40bd41ef0b71d6f7502725dc159fa0d6bebd8bb3 -F src/btreeInt.h 2ad754dd4528baa8d0946a593cc373b890bf859e +F src/btree.c a00a7c4809d642bf6c63979e73face9fed53dc66 +F src/btree.h 00d4cdb747c4172a5566faf037116985dbbc377e +F src/btreeInt.h 9f77bac260d81cc74ecd96cccf73a8c5af1688c4 F src/build.c 28c15c43eefc0066ff64040526ff649c32fe5523 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f @@ -315,8 +315,8 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 388c023582b17890f10c980b30ec1922b471753b F src/os_win.c 40b3af7a47eb1107d0d69e592bec345a3b7b798a F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 18aa67541bd2c7d7963f180669ea4e48cd5246f4 -F src/pager.h 41938212a5d80a7068883bb29ea6ac0d4ef3a6d8 +F src/pager.c fe82e368600278cfd35024c20b888e945236f675 +F src/pager.h 3fc23bca2e7d606ca4fbdd923713cdcdcec48297 F src/parse.y 199145cd982587d70fa3db2f9b11f7ab118f0acd F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0 F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9 @@ -395,15 +395,15 @@ F src/vdbe.c fd6832408d9621ad347a65b01a05380fe766232f F src/vdbe.h 7a75045d879118b9d3af7e8b3c108f2f27c51473 F src/vdbeInt.h 8b54e01ad0463590e7cffabce0bc36da9ee4f816 F src/vdbeapi.c adabbd66eb2e3a10f3998485ee0be7e326d06ee4 -F src/vdbeaux.c 24a03ccbadc8d4cfecf25b6399db95851b10dd21 +F src/vdbeaux.c 27feae106f3f4db298e65d37a9da4962aaff724b F src/vdbeblob.c 4f2e8e075d238392df98c5e03a64342465b03f90 F src/vdbemem.c ae38a0d35ae71cf604381a887c170466ba518090 F src/vdbesort.c f5009e7a35e3065635d8918b9a31f498a499976b F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 F src/vtab.c 082b35a25a26e3d36f365ca8cd73c1922532f05e F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb -F src/wal.c 4497b466fee8ed56c31672bb35741dd0321ec999 -F src/wal.h cd7ea52439d56571d549b7d4eacdeed58d841385 +F src/wal.c 8bb1130dc1876e1c1c3fa082a5b0ec6437feb8e8 +F src/wal.h 5aaed8ca6cad1406088042ff8767951a6a3c7b56 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c 909eba3b6db984eb2adfbca9de2c237ee7056adb F src/whereInt.h 5f87e3c4b0551747d119730dfebddd3c54f04047 @@ -1215,7 +1215,8 @@ F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unique2.test 41e7f83c6827605991160a31380148a9fc5f1339 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 -F test/unlocked.test 644a84d7f032ca029d490f0b0bc8fb9f86c7a1a5 +F test/unlocked.test d143a871bd874311d4e5c98ce458218ca6b579fd +F test/unlocked2.test fc4c730a44c3650250467ecacca6ddb969f64405 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b @@ -1367,7 +1368,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0b9718426e44df092850c5d095ce1b84a1e567cf -R ac1d3857424ab06ed70d671704c3737d +P de1ea450db33b140b11af5b801ea6a15875e774e +R db05422c8fb37e882a7e6cc4bd7c8ffb U dan -Z 12da9addf6141c212c3c7cdffe33d767 +Z cccfd407eb244cf1cd3b5cab72e65fda diff --git a/manifest.uuid b/manifest.uuid index 6a4d41072d..7d70f3c411 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -de1ea450db33b140b11af5b801ea6a15875e774e \ No newline at end of file +3bbc31d515ba9fc920c5cbc7059d3eb1ba9c7f8e \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 7f9054f309..cc97fd8623 100644 --- a/src/btree.c +++ b/src/btree.c @@ -439,6 +439,152 @@ static void downgradeAllSharedCacheTableLocks(Btree *p){ #endif /* SQLITE_OMIT_SHARED_CACHE */ +/* +** The following structure stores the in-memory pointer map used for newly +** allocated pages in UNLOCKED transactions. Such pages are always allocated +** in a contiguous block (from the end of the file) starting with page +** BtreePtrmap.iFirst. +** +** The page number for the parent page iFirst is stored in aPtr[0]. For +** (iFirst+1), aPtr[1]. A zero value indicates that the page has not +** been allocated. +** +** +*/ + +typedef struct RollbackEntry RollbackEntry; +typedef struct PtrmapEntry PtrmapEntry; +struct PtrmapEntry { + Pgno parent; + u8 eType; +}; +struct RollbackEntry { + Pgno pgno; + Pgno parent; + u8 eType; +}; + +struct BtreePtrmap { + Pgno iFirst; /* First new page number aPtr[0] */ + + int nPtrAlloc; /* Allocated size of aPtr[] array */ + PtrmapEntry *aPtr; /* Array of parent page numbers */ + + int nSvpt; /* Used size of aSvpt[] array */ + int nSvptAlloc; /* Allocated size of aSvpt[] */ + int *aSvpt; /* First aRollback[] entry for savepoint i */ + + int nRollback; /* Used size of aRollback[] array */ + int nRollbackAlloc; /* Allocated size of aRollback[] array */ + RollbackEntry *aRollback; /* Array of rollback entries */ +}; + +static int btreePtrmapStore( + BtreePtrmap *pMap, + Pgno pgno, + u8 eType, + Pgno parent +){ + if( pgno>=pMap->iFirst ){ + int iEntry = pgno - pMap->iFirst; + + /* Grow the aPtr[] array if required */ + if( iEntry>=pMap->nPtrAlloc ){ + int nNew = pMap->nPtrAlloc ? pMap->nPtrAlloc*2 : 16; + PtrmapEntry *aNew = (PtrmapEntry*)sqlite3_realloc( + pMap->aPtr, nNew*sizeof(PtrmapEntry) + ); + if( aNew==0 ){ + return SQLITE_NOMEM; + }else{ + int nByte = (nNew-pMap->nPtrAlloc)*sizeof(PtrmapEntry); + memset(&aNew[pMap->nPtrAlloc], 0, nByte); + pMap->aPtr = aNew; + pMap->nPtrAlloc = nNew; + } + } + + /* Add an entry to the rollback log if required */ + if( pMap->nSvpt>0 && pMap->aPtr[iEntry].parent ){ + if( pMap->nRollback>=pMap->nRollbackAlloc ){ + int nNew = pMap->nRollback ? pMap->nRollback*2 : 16; + RollbackEntry *aNew = (RollbackEntry*)sqlite3_realloc( + pMap->aRollback, nNew*sizeof(RollbackEntry) + ); + if( aNew==0 ){ + return SQLITE_NOMEM; + }else{ + pMap->aRollback = aNew; + pMap->nRollbackAlloc = nNew; + } + } + + pMap->aRollback[pMap->nRollback].pgno = pgno; + pMap->aRollback[pMap->nRollback].parent = pMap->aPtr[iEntry].parent; + pMap->aRollback[pMap->nRollback].eType = pMap->aPtr[iEntry].eType; + } + + /* Update the aPtr[] array */ + pMap->aPtr[iEntry].parent = parent; + pMap->aPtr[iEntry].eType = eType; + } + + return SQLITE_OK; +} + +/* +** Open savepoint iSavepoint, if it is not already open. +*/ +static int btreePtrmapBegin(BtreePtrmap *pMap, int nSvpt){ + if( nSvptnSvpt ){ + int i; + if( nSvpt>=pMap->nSvptAlloc ){ + int nNew = pMap->nSvptAlloc ? pMap->nSvptAlloc*2 : 16; + int *aNew = sqlite3_realloc(pMap->aSvpt, sizeof(int) * nNew); + if( aNew==0 ){ + return SQLITE_NOMEM; + }else{ + pMap->aSvpt = aNew; + pMap->nSvptAlloc = nNew; + } + } + + for(i=pMap->nSvpt; iaSvpt[i] = pMap->nRollback; + } + pMap->nSvpt = nSvpt; + } + + return SQLITE_OK; +} + +/* +** Rollback (if op==SAVEPOINT_ROLLBACK) or release (if op==SAVEPOINT_RELEASE) +** savepoint iSvpt. +*/ +static void btreePtrmapEnd(BtreePtrmap *pMap, int op, int iSvpt){ + assert( op==SAVEPOINT_ROLLBACK || op==SAVEPOINT_RELEASE ); + assert( iSvpt>=0 || (iSvpt==-1 && op==SAVEPOINT_ROLLBACK) ); + if( iSvpt<0 ){ + pMap->nSvpt = 0; + pMap->nRollback = 0; + memset(pMap->aPtr, 0, sizeof(Pgno) * pMap->nPtrAlloc); + }else if( iSvptnSvpt ){ + if( op==SAVEPOINT_ROLLBACK ){ + int ii; + for(ii=pMap->nRollback-1; ii>=pMap->aSvpt[iSvpt]; ii--){ + RollbackEntry *p = &pMap->aRollback[ii]; + PtrmapEntry *pEntry = &pMap->aPtr[p->pgno - pMap->iFirst]; + pEntry->parent = p->parent; + pEntry->eType = p->eType; + } + } + pMap->nSvpt = iSvpt + (op==SAVEPOINT_ROLLBACK); + pMap->nRollback = pMap->aSvpt[iSvpt]; + } +} + + static void releasePage(MemPage *pPage); /* Forward reference */ /* @@ -878,42 +1024,44 @@ static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){ int offset; /* Offset in pointer map page */ int rc; /* Return code from subfunctions */ + assert( sqlite3_mutex_held(pBt->mutex) ); if( *pRC ) return; - assert( sqlite3_mutex_held(pBt->mutex) ); - /* The master-journal page number must never be used as a pointer map page */ - assert( 0==PTRMAP_ISPAGE(pBt, PENDING_BYTE_PAGE(pBt)) ); + if( pBt->pMap ){ + *pRC = btreePtrmapStore(pBt->pMap, key, eType, parent); + }else{ + /* The master-journal page number must never be used as a ptr map page */ + assert( 0==PTRMAP_ISPAGE(pBt, PENDING_BYTE_PAGE(pBt)) ); - assert( pBt->autoVacuum ); - if( key==0 ){ - *pRC = SQLITE_CORRUPT_BKPT; - return; - } - iPtrmap = PTRMAP_PAGENO(pBt, key); - rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage); - if( rc!=SQLITE_OK ){ - *pRC = rc; - return; - } - offset = PTRMAP_PTROFFSET(iPtrmap, key); - if( offset<0 ){ - *pRC = SQLITE_CORRUPT_BKPT; - goto ptrmap_exit; - } - assert( offset <= (int)pBt->usableSize-5 ); - pPtrmap = (u8 *)sqlite3PagerGetData(pDbPage); + assert( pBt->autoVacuum ); + if( key==0 ){ + *pRC = SQLITE_CORRUPT_BKPT; + return; + } + iPtrmap = PTRMAP_PAGENO(pBt, key); + rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage); + if( rc!=SQLITE_OK ){ + *pRC = rc; + return; + } + offset = PTRMAP_PTROFFSET(iPtrmap, key); + if( offset<0 ){ + *pRC = SQLITE_CORRUPT_BKPT; + }else{ + assert( offset <= (int)pBt->usableSize-5 ); + pPtrmap = (u8 *)sqlite3PagerGetData(pDbPage); - if( eType!=pPtrmap[offset] || get4byte(&pPtrmap[offset+1])!=parent ){ - TRACE(("PTRMAP_UPDATE: %d->(%d,%d)\n", key, eType, parent)); - *pRC= rc = sqlite3PagerWrite(pDbPage); - if( rc==SQLITE_OK ){ - pPtrmap[offset] = eType; - put4byte(&pPtrmap[offset+1], parent); + if( eType!=pPtrmap[offset] || get4byte(&pPtrmap[offset+1])!=parent ){ + TRACE(("PTRMAP_UPDATE: %d->(%d,%d)\n", key, eType, parent)); + *pRC= rc = sqlite3PagerWrite(pDbPage); + if( rc==SQLITE_OK ){ + pPtrmap[offset] = eType; + put4byte(&pPtrmap[offset+1], parent); + } + } } + sqlite3PagerUnref(pDbPage); } - -ptrmap_exit: - sqlite3PagerUnref(pDbPage); } /* @@ -1924,6 +2072,50 @@ u32 sqlite3BtreeLastPage(Btree *p){ return btreePagecount(p->pBt); } +#ifdef SQLITE_ENABLE_UNLOCKED +/* +** This function is called before allocating or freeing a b-tree page. If +** the current transaction is UNLOCKED, it allocates the BtreePtrmap +** structure and zeroes the nFree/iTrunk fields in the database header +** on page 1. +*/ +static int allocatePtrmap(BtShared *pBt){ + int rc = SQLITE_OK; + if( pBt->pMap==0 && sqlite3PagerIsUnlocked(pBt->pPager) ){ + /* If this is an unlocked transaction, set the header values + ** identifying the size of the free-list and the page number + ** of the first trunk page to zero. */ + BtreePtrmap *pMap = sqlite3_malloc(sizeof(BtreePtrmap)); + if( pMap==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(&pBt->pPage1->aData[32], 0, sizeof(u32)*2); + memset(pMap, 0, sizeof(BtreePtrmap)); + pMap->iFirst = btreePagecount(pBt) + 1; + pBt->pMap = pMap; + } + } + return rc; +} + +/* +** Free a pointer-map allocated by allocatePtrmap. +*/ +static void deletePtrmap(BtShared *pBt){ + BtreePtrmap *pMap = pBt->pMap; + if( pMap ){ + sqlite3_free(pMap->aRollback); + sqlite3_free(pMap->aPtr); + sqlite3_free(pMap->aSvpt); + sqlite3_free(pMap); + pBt->pMap = 0; + } +} +#else +#define allocatePtrmap(x) SQLITE_OK +#define deletePtrmap(x) +#endif + /* ** Get a page from the pager and initialize it. ** @@ -3149,6 +3341,9 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ if( rc==SQLITE_OK ){ rc = newDatabase(pBt); } + if( rc==SQLITE_OK ){ + rc = allocatePtrmap(pBt); + } } } @@ -3205,7 +3400,11 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ ** open savepoints. If the second parameter is greater than 0 and ** the sub-journal is not already open, then it will be opened here. */ - rc = sqlite3PagerOpenSavepoint(pBt->pPager, p->db->nSavepoint); + int nSavepoint = p->db->nSavepoint; + rc = sqlite3PagerOpenSavepoint(pBt->pPager, nSavepoint); + if( pBt->pMap && rc==SQLITE_OK && nSavepoint ){ + rc = btreePtrmapBegin(pBt->pMap, nSavepoint); + } } btreeIntegrity(p); @@ -3642,6 +3841,92 @@ static int autoVacuumCommit(BtShared *pBt){ # define setChildPtrmaps(x) SQLITE_OK #endif +/* +** The b-tree handle passed as the only argument is about to commit an +** UNLOCKED transaction. At this point it is guaranteed that this is +** possible - the wal WRITER lock is held and it is known that there are +** no conflicts with committed transactions. +*/ +static int btreeFixUnlocked(Btree *p){ + BtShared *pBt = p->pBt; + MemPage *pPage1 = pBt->pPage1; + u8 *p1 = pPage1->aData; + Pager *pPager = pBt->pPager; + int rc = SQLITE_OK; + + /* If page 1 of the database is not writable, then no pages were allocated + ** or freed by this transaction. In this case no special handling is + ** required. Otherwise, if page 1 is dirty, proceed. */ + BtreePtrmap *pMap = pBt->pMap; + Pgno iTrunk = get4byte(&p1[32]); + Pgno nPage = btreePagecount(pBt); + Pgno nOrig = pMap->iFirst-1; + u32 nFree = get4byte(&p1[36]); + + assert( sqlite3PagerIsUnlocked(pPager) ); + assert( pBt->pMap ); + rc = sqlite3PagerUpgradeSnapshot(pPager, pPage1->pDbPage); + assert( p1==pPage1->aData ); + + if( rc==SQLITE_OK ){ + Pgno nHPage = get4byte(&p1[28]); + Pgno nFinal = nHPage; + + if( sqlite3PagerIswriteable(pPage1->pDbPage) ){ + Pgno iHTrunk = get4byte(&p1[32]); + u32 nHFree = get4byte(&p1[36]); + + /* Attach the head database free list to the end of the current + ** transactions free-list (if any). */ + if( iTrunk!=0 ){ + put4byte(&p1[36], nHFree + nFree); + put4byte(&p1[32], iTrunk); + while( iTrunk ){ + DbPage *pTrunk = sqlite3PagerLookup(pPager, iTrunk); + iTrunk = get4byte((u8*)pTrunk->pData); + if( iTrunk==0 ){ + put4byte((u8*)pTrunk->pData, iHTrunk); + } + sqlite3PagerUnref(pTrunk); + }; + } + + if( nHPageiFirst through + ** nPage (inclusive) at the end of the database file. Meanwhile, + ** other transactions have allocated (iFirst..nHPage). So move + ** pages (iFirst..MIN(nPage,nHPage)) to (MAX(nPage,nHPage)+1). + */ + Pgno iLast = MIN(nPage, nHPage); /* Last page to move */ + Pgno iPg; + + nFinal = MAX(nPage, nHPage); /* Final size of database */ + for(iPg=pMap->iFirst; iPg<=iLast && rc==SQLITE_OK; iPg++){ + MemPage *pPg = 0; + Pgno iNew; /* New page number for pPg */ + PtrmapEntry *pEntry; /* Pointer map entry for page iPg */ + + btreeGetPage(pBt, iPg, &pPg, 0); + assert( sqlite3PagerIswriteable(pPg->pDbPage) ); + assert( sqlite3PagerPageRefcount(pPg->pDbPage)==1 ); + pEntry = &pMap->aPtr[iPg - pMap->iFirst]; + iNew = ++nFinal; + + rc = relocatePage(pBt, pPg, pEntry->eType, pEntry->parent, iNew, 1); + releasePageNotNull(pPg); + } + + put4byte(&p1[28], nFinal); + } + } + sqlite3PagerSetDbsize(pPager, nFinal); + } + + return rc; +} + /* ** This routine does the first phase of a two-phase commit. This routine ** causes a rollback journal to be created (if it does not already exist) @@ -3678,10 +3963,8 @@ int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zMaster){ /* Figure out if this is a commit of an UNLOCKED transaction that ** requires a snapshot upgrade. If so, skip any auto-vacuum ** processing. */ - if( pBt->autoVacuum && ( - 0==pBt->db->bUnlocked - || 0==sqlite3PagerCommitRequiresUpgrade(pBt->pPager) - )){ + if( pBt->autoVacuum ){ + assert( sqlite3PagerIsUnlocked(pBt->pPager)==0 ); rc = autoVacuumCommit(pBt); if( rc!=SQLITE_OK ){ sqlite3BtreeLeave(p); @@ -3692,6 +3975,9 @@ int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zMaster){ sqlite3PagerTruncateImage(pBt->pPager, pBt->nPage); } #endif + if( rc==SQLITE_OK && sqlite3PagerIsUnlocked(pBt->pPager) ){ + rc = btreeFixUnlocked(p); + } if( rc==SQLITE_OK ){ rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zMaster, 0); } @@ -3737,6 +4023,8 @@ static void btreeEndTransaction(Btree *p){ unlockBtreeIfUnused(pBt); } + /* If this was an UNLOCKED transaction, delete the pBt->pMap object */ + deletePtrmap(pBt); btreeIntegrity(p); } @@ -3962,6 +4250,9 @@ int sqlite3BtreeBeginStmt(Btree *p, int iStatement){ ** such savepoints while the statement transaction savepoint is active. */ rc = sqlite3PagerOpenSavepoint(pBt->pPager, iStatement); + if( rc==SQLITE_OK && pBt->pMap ){ + rc = btreePtrmapBegin(pBt->pMap, iStatement); + } sqlite3BtreeLeave(p); return rc; } @@ -3985,6 +4276,7 @@ int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK ); assert( iSavepoint>=0 || (iSavepoint==-1 && op==SAVEPOINT_ROLLBACK) ); sqlite3BtreeEnter(p); + if( pBt->pMap ) btreePtrmapEnd(pBt->pMap, op, iSavepoint); rc = sqlite3PagerSavepoint(pBt->pPager, op, iSavepoint); if( rc==SQLITE_OK ){ if( iSavepoint<0 && (pBt->btsFlags & BTS_INITIALLY_EMPTY)!=0 ){ @@ -5493,6 +5785,16 @@ static int allocateBtreePage( if( n>=mxPage ){ return SQLITE_CORRUPT_BKPT; } + + /* Ensure page 1 is writable. This function will either change the number + ** of pages in the free-list or the size of the database file. Since both + ** of these operations involve modifying page 1 header fields, page 1 + ** will definitely be written by this transaction. If this is an UNLOCKED + ** transaction, ensure the BtreePtrmap structure has been allocated. */ + assert( eMode!=BTALLOC_EXACT || sqlite3PagerIsUnlocked(pBt->pPager)==0 ); + rc = sqlite3PagerWrite(pPage1->pDbPage); + if( rc ) return rc; + if( n>0 ){ /* There are pages on the freelist. Reuse one of those pages. */ Pgno iTrunk; @@ -5523,8 +5825,6 @@ static int allocateBtreePage( /* Decrement the free-list count by 1. Set iTrunk to the index of the ** first free-list trunk page. iPrevTrunk is initially 1. */ - rc = sqlite3PagerWrite(pPage1->pDbPage); - if( rc ) return rc; put4byte(&pPage1->aData[36], n-1); /* The code within this loop is run only once if the 'searchList' variable @@ -5830,7 +6130,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ /* If the database supports auto-vacuum, write an entry in the pointer-map ** to indicate that the page is free. */ - if( ISAUTOVACUUM ){ + if( REQUIRE_PTRMAP ){ ptrmapPut(pBt, iPage, PTRMAP_FREEPAGE, 0, &rc); if( rc ) goto freepage_out; } @@ -6116,7 +6416,7 @@ static int fillInCell( } #endif rc = allocateBtreePage(pBt, &pOvfl, &pgnoOvfl, pgnoOvfl, 0); -#ifndef SQLITE_OMIT_AUTOVACUUM + /* If the database supports auto-vacuum, and the second or subsequent ** overflow page is being allocated, add an entry to the pointer-map ** for that page now. @@ -6127,14 +6427,13 @@ static int fillInCell( ** may misinterpret the uninitialized values and delete the ** wrong pages from the database. */ - if( pBt->autoVacuum && rc==SQLITE_OK ){ + if( REQUIRE_PTRMAP && rc==SQLITE_OK ){ u8 eType = (pgnoPtrmap?PTRMAP_OVERFLOW2:PTRMAP_OVERFLOW1); ptrmapPut(pBt, pgnoOvfl, eType, pgnoPtrmap, &rc); if( rc ){ releasePage(pOvfl); } } -#endif if( rc ){ releasePage(pToRelease); return rc; @@ -6301,6 +6600,7 @@ static void insertCell( assert( j==0 || pPage->aiOvfl[j-1]<(u16)i ); /* Overflows in sorted order */ assert( j==0 || i==pPage->aiOvfl[j-1]+1 ); /* Overflows are sequential */ }else{ + BtShared *pBt = pPage->pBt; int rc = sqlite3PagerWrite(pPage->pDbPage); if( rc!=SQLITE_OK ){ *pRC = rc; @@ -6315,7 +6615,7 @@ static void insertCell( ** if it returns successfully */ assert( idx >= 0 ); assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB ); - assert( idx+sz <= (int)pPage->pBt->usableSize ); + assert( idx+sz <= (int)pBt->usableSize ); pPage->nFree -= (u16)(2 + sz); memcpy(&data[idx], pCell, sz); if( iChild ){ @@ -6328,14 +6628,12 @@ static void insertCell( /* increment the cell count */ if( (++data[pPage->hdrOffset+4])==0 ) data[pPage->hdrOffset+3]++; assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell ); -#ifndef SQLITE_OMIT_AUTOVACUUM - if( pPage->pBt->autoVacuum ){ + if( REQUIRE_PTRMAP ){ /* The cell may contain a pointer to an overflow page. If so, write ** the entry for the overflow page into the pointer map. */ ptrmapPutOvflPtr(pPage, pCell, pRC); } -#endif } } @@ -6740,7 +7038,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ ** be marked as dirty. Returning an error code will cause a ** rollback, undoing any changes made to the parent page. */ - if( ISAUTOVACUUM ){ + if( REQUIRE_PTRMAP ){ ptrmapPut(pBt, pgnoNew, PTRMAP_BTREE, pParent->pgno, &rc); if( szCell>pNew->minLocal ){ ptrmapPutOvflPtr(pNew, pCell, &rc); @@ -6875,7 +7173,7 @@ static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){ /* If this is an auto-vacuum database, update the pointer-map entries ** for any b-tree or overflow pages that pTo now contains the pointers to. */ - if( ISAUTOVACUUM ){ + if( REQUIRE_PTRMAP ){ *pRC = setChildPtrmaps(pTo); } } @@ -7345,7 +7643,7 @@ static int balance_nonroot( cntOld[i] = b.nCell; /* Set the pointer-map entry for the new sibling page. */ - if( ISAUTOVACUUM ){ + if( REQUIRE_PTRMAP ){ ptrmapPut(pBt, pNew->pgno, PTRMAP_BTREE, pParent->pgno, &rc); if( rc!=SQLITE_OK ){ goto balance_cleanup; @@ -7439,7 +7737,7 @@ static int balance_nonroot( ** updated. This happens below, after the sibling pages have been ** populated, not here. */ - if( ISAUTOVACUUM ){ + if( REQUIRE_PTRMAP ){ MemPage *pNew = apNew[0]; u8 *aOld = pNew->aData; int cntOldNext = pNew->nCell + pNew->nOverflow; @@ -7624,7 +7922,7 @@ static int balance_nonroot( ); copyNodeContent(apNew[0], pParent, &rc); freePage(apNew[0], &rc); - }else if( ISAUTOVACUUM && !leafCorrection ){ + }else if( REQUIRE_PTRMAP && !leafCorrection ){ /* Fix the pointer map entries associated with the right-child of each ** sibling page. All other pointer map entries have already been taken ** care of. */ @@ -7710,7 +8008,7 @@ static int balance_deeper(MemPage *pRoot, MemPage **ppChild){ if( rc==SQLITE_OK ){ rc = allocateBtreePage(pBt,&pChild,&pgnoChild,pRoot->pgno,0); copyNodeContent(pRoot, pChild, &rc); - if( ISAUTOVACUUM ){ + if( REQUIRE_PTRMAP ){ ptrmapPut(pBt, pgnoChild, PTRMAP_BTREE, pRoot->pgno, &rc); } } @@ -9590,3 +9888,13 @@ int sqlite3BtreeIsReadonly(Btree *p){ ** Return the size of the header added to each page by this module. */ int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); } + +int sqlite3BtreeExclusiveLock(Btree *p){ + int rc; + BtShared *pBt = p->pBt; + assert( p->inTrans==TRANS_WRITE && pBt->pPage1 ); + sqlite3BtreeEnter(p); + rc = sqlite3PagerExclusiveLock(pBt->pPager, pBt->pPage1->pDbPage); + sqlite3BtreeLeave(p); + return rc; +} diff --git a/src/btree.h b/src/btree.h index 50dc8ca290..8f611b7a9b 100644 --- a/src/btree.h +++ b/src/btree.h @@ -215,6 +215,8 @@ int sqlite3BtreeCursorHasHint(BtCursor*, unsigned int mask); int sqlite3BtreeIsReadonly(Btree *pBt); int sqlite3HeaderSizeBtree(void); +int sqlite3BtreeExclusiveLock(Btree *pBt); + #ifndef NDEBUG int sqlite3BtreeCursorIsValid(BtCursor*); #endif diff --git a/src/btreeInt.h b/src/btreeInt.h index cbf6c99847..67b1da6e3f 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -232,6 +232,7 @@ typedef struct MemPage MemPage; typedef struct BtLock BtLock; typedef struct CellInfo CellInfo; +typedef struct BtreePtrmap BtreePtrmap; /* ** This is a magic string that appears at the beginning of every @@ -447,6 +448,9 @@ struct BtShared { Btree *pWriter; /* Btree with currently open write transaction */ #endif u8 *pTmpSpace; /* Temp space sufficient to hold a single cell */ +#ifdef SQLITE_ENABLE_UNLOCKED + BtreePtrmap *pMap; +#endif }; /* @@ -659,6 +663,12 @@ struct BtCursor { #define ISAUTOVACUUM 0 #endif +#ifdef SQLITE_ENABLE_UNLOCKED +# define REQUIRE_PTRMAP (ISAUTOVACUUM || pBt->pMap) +#else +# define REQUIRE_PTRMAP ISAUTOVACUUM +#endif + /* ** This structure is passed around through all the sanity checking routines diff --git a/src/pager.c b/src/pager.c index 5107edda36..efbea0ab8e 100644 --- a/src/pager.c +++ b/src/pager.c @@ -657,7 +657,9 @@ struct Pager { u32 cksumInit; /* Quasi-random value added to every checksum */ u32 nSubRec; /* Number of records written to sub-journal */ Bitvec *pInJournal; /* One bit for each page in the database file */ +#ifdef SQLITE_ENABLE_UNLOCKED Bitvec *pAllRead; /* Pages read within current UNLOCKED trans. */ +#endif sqlite3_file *fd; /* File descriptor for database */ sqlite3_file *jfd; /* File descriptor for main journal */ sqlite3_file *sjfd; /* File descriptor for sub-journal */ @@ -4091,7 +4093,7 @@ static int syncJournal(Pager *pPager, int newHdr){ assert( assert_pager_state(pPager) ); assert( !pagerUseWal(pPager) ); - rc = sqlite3PagerExclusiveLock(pPager); + rc = sqlite3PagerExclusiveLock(pPager, 0); if( rc!=SQLITE_OK ) return rc; if( !pPager->noSync ){ @@ -5929,11 +5931,9 @@ int sqlite3PagerWrite(PgHdr *pPg){ ** to sqlite3PagerWrite(). In other words, return TRUE if it is ok ** to change the content of the page. */ -#ifndef NDEBUG int sqlite3PagerIswriteable(DbPage *pPg){ return pPg->flags & PGHDR_WRITEABLE; } -#endif /* ** A call to this routine tells the pager that it is not necessary to @@ -6091,7 +6091,7 @@ int sqlite3PagerSync(Pager *pPager, const char *zMaster){ ** Otherwise, either SQLITE_BUSY or an SQLITE_IOERR_XXX error code is ** returned. */ -int sqlite3PagerExclusiveLock(Pager *pPager){ +int sqlite3PagerExclusiveLock(Pager *pPager, PgHdr *pPage1){ int rc = SQLITE_OK; assert( pPager->eState==PAGER_WRITER_CACHEMOD || pPager->eState==PAGER_WRITER_DBMOD @@ -6108,7 +6108,7 @@ int sqlite3PagerExclusiveLock(Pager *pPager){ ** non-zero. */ do { /* rc = sqlite3WalBeginWriteTransaction(pWal); */ - rc = sqlite3WalLockForCommit(pPager->pWal, pPager->pAllRead); + rc = sqlite3WalLockForCommit(pPager->pWal, pPage1, pPager->pAllRead); }while( rc==SQLITE_BUSY && pPager->xBusyHandler(pPager->pBusyHandlerArg) ); @@ -6117,6 +6117,24 @@ int sqlite3PagerExclusiveLock(Pager *pPager){ return rc; } +int sqlite3PagerUpgradeSnapshot(Pager *pPager, DbPage *pPage1){ + int rc; + u32 iFrame = 0; + + assert( pPager->pWal && pPager->pAllRead ); + sqlite3WalUpgradeSnapshot(pPager->pWal); + rc = sqlite3WalFindFrame(pPager->pWal, 1, &iFrame); + if( rc==SQLITE_OK ){ + rc = readDbPage(pPage1, iFrame); + } + + return rc; +} + +void sqlite3PagerSetDbsize(Pager *pPager, Pgno nFinal){ + pPager->dbSize = nFinal; +} + /* ** If this is a WAL mode connection and the WRITER lock is currently held, ** relinquish it. @@ -6131,14 +6149,11 @@ void sqlite3PagerDropExclusiveLock(Pager *pPager){ ** Return true if this is a WAL database and snapshot upgrade is required ** before the current transaction can be committed. */ -int sqlite3PagerCommitRequiresUpgrade(Pager *pPager){ - int res = 0; - if( pagerUseWal(pPager) ){ - res = sqlite3WalCommitRequiresUpgrade(pPager->pWal); - } - return res; +int sqlite3PagerIsUnlocked(Pager *pPager){ + return pPager->pAllRead!=0; } + /* ** Sync the database file for the pager pPager. zMaster points to the name ** of a master journal file that should be written into the individual diff --git a/src/pager.h b/src/pager.h index 3c0b9a1a71..92e5546282 100644 --- a/src/pager.h +++ b/src/pager.h @@ -145,12 +145,13 @@ int sqlite3PagerMovepage(Pager*,DbPage*,Pgno,int); int sqlite3PagerPageRefcount(DbPage*); void *sqlite3PagerGetData(DbPage *); void *sqlite3PagerGetExtra(DbPage *); +int sqlite3PagerIsDirty(DbPage*); /* Functions used to manage pager transactions and savepoints. */ void sqlite3PagerPagecount(Pager*, int*); int sqlite3PagerBegin(Pager*, int exFlag, int); int sqlite3PagerCommitPhaseOne(Pager*,const char *zMaster, int); -int sqlite3PagerExclusiveLock(Pager*); +int sqlite3PagerExclusiveLock(Pager*, DbPage *pPage1); int sqlite3PagerSync(Pager *pPager, const char *zMaster); int sqlite3PagerCommitPhaseTwo(Pager*); int sqlite3PagerRollback(Pager*); @@ -159,7 +160,7 @@ int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint); int sqlite3PagerSharedLock(Pager *pPager); void sqlite3PagerDropExclusiveLock(Pager*); -int sqlite3PagerCommitRequiresUpgrade(Pager*); +int sqlite3PagerIsUnlocked(Pager*); #ifndef SQLITE_OMIT_WAL int sqlite3PagerCheckpoint(Pager *pPager, int, int*, int*); @@ -196,6 +197,10 @@ void sqlite3PagerTruncateImage(Pager*,Pgno); void sqlite3PagerRekey(DbPage*, Pgno, u16); +int sqlite3PagerIswriteable(DbPage*); +int sqlite3PagerUpgradeSnapshot(Pager *pPager, DbPage*); +void sqlite3PagerSetDbsize(Pager *pPager, Pgno); + #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) void *sqlite3PagerCodec(DbPage *); #endif @@ -203,7 +208,6 @@ void *sqlite3PagerCodec(DbPage *); /* Functions to support testing and debugging. */ #if !defined(NDEBUG) || defined(SQLITE_TEST) Pgno sqlite3PagerPagenumber(DbPage*); - int sqlite3PagerIswriteable(DbPage*); #endif #ifdef SQLITE_TEST int *sqlite3PagerStats(Pager*); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 93a7187ab9..f5f9d0fbdd 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2020,9 +2020,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ if( sqlite3BtreeIsInTrans(pBt) ){ needXcommit = 1; if( i!=1 ) nTrans++; - sqlite3BtreeEnter(pBt); - rc = sqlite3PagerExclusiveLock(sqlite3BtreePager(pBt)); - sqlite3BtreeLeave(pBt); + rc = sqlite3BtreeExclusiveLock(pBt); } } diff --git a/src/wal.c b/src/wal.c index 5fbdae6a1a..809ce2b109 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2573,7 +2573,8 @@ int sqlite3WalBeginWriteTransaction(Wal *pWal){ ** transaction. It may be assumed that no frames have been written to ** the wal file. */ -int sqlite3WalLockForCommit(Wal *pWal, Bitvec *pAllRead){ +int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPage1, Bitvec *pAllRead){ + Pager *pPager = pPage1->pPager; int rc = walWriteLock(pWal); /* If the database has been modified since this transaction was started, @@ -2589,6 +2590,9 @@ int sqlite3WalLockForCommit(Wal *pWal, Bitvec *pAllRead){ if( rc==SQLITE_OK ){ volatile WalIndexHdr *pHead; /* Head of the wal file */ pHead = walIndexHdr(pWal); + + /* TODO: Check header checksum is good here. */ + if( memcmp(&pWal->hdr, (void*)pHead, sizeof(WalIndexHdr))!=0 ){ /* TODO: Is this safe? Because it holds the WRITER lock this thread ** has exclusive access to the live header, but might it be corrupt? @@ -2609,12 +2613,33 @@ int sqlite3WalLockForCommit(Wal *pWal, Bitvec *pAllRead){ if( iMin<1 ) iMin = 1; if( iMax>pHead->mxFrame ) iMax = pHead->mxFrame; for(i=iMin; i<=iMax; i++){ - if( sqlite3BitvecTestNotNull(pAllRead, aPgno[i]) ){ + PgHdr *pPg; + if( aPgno[i]==1 ){ + /* Check that the schema cookie has not been modified. If + ** it has not, the commit can proceed. */ + u8 aNew[4]; + u8 *aOld = &((u8*)pPage1->pData)[40]; + int sz; + i64 iOffset; + sz = pWal->hdr.szPage; + sz = (sz&0xfe00) + ((sz&0x0001)<<16); + iOffset = walFrameOffset(i+iZero, sz) + WAL_FRAME_HDRSIZE + 40; + rc = sqlite3OsRead(pWal->pWalFd, aNew, sizeof(aNew), iOffset); + if( rc==SQLITE_OK && memcmp(aOld, aNew, sizeof(aNew)) ){ + rc = SQLITE_BUSY_SNAPSHOT; + } + }else if( sqlite3BitvecTestNotNull(pAllRead, aPgno[i]) ){ sqlite3_log(SQLITE_OK, "cannot commit UNLOCKED transaction (conflict at page %d)", (int)aPgno[i] ); rc = SQLITE_BUSY_SNAPSHOT; + }else if( (pPg = sqlite3PagerLookup(pPager, aPgno[i])) ){ + if( sqlite3PagerIswriteable(pPg)==0 ){ + sqlite3PcacheDrop(pPg); + }else{ + sqlite3PagerUnref(pPg); + } } } } @@ -2626,6 +2651,11 @@ int sqlite3WalLockForCommit(Wal *pWal, Bitvec *pAllRead){ return rc; } +void sqlite3WalUpgradeSnapshot(Wal *pWal){ + assert( pWal->writeLock ); + memcpy(&pWal->hdr, (void*)walIndexHdr(pWal), sizeof(WalIndexHdr)); +} + /* ** This function is only ever called while committing an UNLOCKED ** transaction, after the caller has already obtained the WRITER lock @@ -2890,7 +2920,6 @@ int sqlite3WalFrames( int szFrame; /* The size of a single frame */ i64 iOffset; /* Next byte to write in WAL file */ WalWriter w; /* The writer */ - int bUpgrade = 0; /* True if commit requires snapshot upgrade */ assert( pList ); assert( pWal->writeLock ); @@ -2906,22 +2935,6 @@ int sqlite3WalFrames( } #endif - if( isCommit ){ - volatile WalIndexHdr *pHead = walIndexHdr(pWal); - if( pHead->mxFrame>pWal->hdr.mxFrame ){ - if( memcmp((void*)&pHead[0], (void*)&pHead[1], sizeof(WalIndexHdr))!=0 ){ - /* TODO: Deal with this case. It's quite possible, but fiddly. */ - return SQLITE_CORRUPT_BKPT; - } - memcpy(&pWal->hdr, (void*)pHead, sizeof(WalIndexHdr)); - if( nTruncatehdr.nPage ){ - /* Do not truncate the database file in this case */ - nTruncate = pWal->hdr.nPage; - } - bUpgrade = 1; - } - } - /* See if it is possible to write these frames into the start of the ** log file, instead of appending to it at pWal->hdr.mxFrame. */ @@ -3070,14 +3083,6 @@ int sqlite3WalFrames( } } - if( rc==SQLITE_OK && bUpgrade ){ - /* If this commit required a snapshot upgrade, the pager cache is - ** not currently consistent with the head of the wal file. Zeroing - ** Wal.hdr here forces the next transaction to reset the cache - ** before beginning to read the db. */ - memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); - } - WALTRACE(("WAL%p: frame write %s\n", pWal, rc ? "failed" : "ok")); return rc; } diff --git a/src/wal.h b/src/wal.h index fe85a52cc0..5b4ca15943 100644 --- a/src/wal.h +++ b/src/wal.h @@ -127,8 +127,9 @@ int sqlite3WalExclusiveMode(Wal *pWal, int op); int sqlite3WalHeapMemory(Wal *pWal); /* Return true if the WRITER lock is held. False otherwise. */ -int sqlite3WalLockForCommit(Wal *pWal, Bitvec *pRead); +int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPg, Bitvec *pRead); int sqlite3WalCommitRequiresUpgrade(Wal *pWal); +void sqlite3WalUpgradeSnapshot(Wal *pWal); #ifdef SQLITE_ENABLE_ZIPVFS /* If the WAL file is not empty, return the number of bytes of content diff --git a/test/unlocked.test b/test/unlocked.test index 23eb5a5064..c743cc6328 100644 --- a/test/unlocked.test +++ b/test/unlocked.test @@ -399,10 +399,9 @@ do_multiclient_test tn { do_test 2.$tn.7.3 { list [catch { sql1 { COMMIT } } msg] $msg [sqlite3_errcode db] - } {1 {database is locked} SQLITE_BUSY_SNAPSHOT} - sql1 ROLLBACK - + } {0 {} SQLITE_OK} + do_test 2.$tn.7.4 { sql3 { PRAGMA integrity_check } } ok } diff --git a/test/unlocked2.test b/test/unlocked2.test new file mode 100644 index 0000000000..f808a84e7b --- /dev/null +++ b/test/unlocked2.test @@ -0,0 +1,101 @@ +# 2015 July 26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set ::testprefix unlocked2 + + +do_multiclient_test tn { + + do_test 1.$tn.1 { + sql1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x); + CREATE TABLE t2(y); + } + } {wal} + + # Test that an UNLOCKED transaction that allocates/frees no pages does + # not conflict with a transaction that does allocate pages. + do_test 1.$tn.2 { + sql1 { + BEGIN UNLOCKED; + INSERT INTO t1 VALUES(4); + } + sql2 { + INSERT INTO t2 VALUES(randomblob(1500)); + } + sql1 { + COMMIT; + } + } {} + + # But that an UNLOCKED transaction does conflict with a transaction + # that modifies the db schema. + do_test 1.$tn.3 { + sql1 { + BEGIN UNLOCKED; + INSERT INTO t1 VALUES(5); + } + sql2 { + CREATE TABLE t3(z); + } + list [catch { sql1 COMMIT } msg] $msg + } {1 {database is locked}} + + # Test that an UNLOCKED transaction that allocates at least one page + # does not conflict with a transaction that allocates no pages. + do_test 1.$tn.4 { + sql1 { + ROLLBACK; + BEGIN UNLOCKED; + INSERT INTO t1 VALUES(randomblob(1500)); + } + sql2 { + INSERT INTO t2 VALUES(8); + } + sql1 { + COMMIT; + } + } {} + + do_test 1.$tn.5 { sql3 { PRAGMA integrity_check } } {ok} +} + +do_multiclient_test tn { + do_test 2.$tn.1 { + sql1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x UNIQUE); + CREATE TABLE t2(y UNIQUE); + } + } {wal} + + do_test 2.$tn.2 { + sql1 { + BEGIN UNLOCKED; + INSERT INTO t1 VALUES(randomblob(1500)); + } + sql2 { + INSERT INTO t2 VALUES(randomblob(1500)); + } + sql1 { + COMMIT; + } + } {} + + do_test 2.$tn.3 { sql3 { PRAGMA integrity_check } } {ok} +} + +finish_test From 699bdf056b62070d2cf5d47b8678ddf4bcb72479 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 20 Aug 2015 20:25:56 +0000 Subject: [PATCH 006/179] Fix a problem causing corruption when an UNLOCKED transaction is rolled back. FossilOrigin-Name: 7c36147846484c96023939864417b5624f3bc5f8 --- manifest | 14 +++++----- manifest.uuid | 2 +- src/pager.c | 9 +++++++ test/unlocked2.test | 64 +++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 79 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 92f5317a3d..ba7247bf36 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\scommitting\san\sunlocked\stransaction,\srelocate\snewly\sallocated\sdatabase\spages\swithin\sthe\sfile\sto\savoid\sconflicting\swith\scommitted\stransactions.\sThere\sare\slots\sof\sthings\sstill\sto\sfix\sin\sthis\scode. -D 2015-08-19T20:27:05.840 +C Fix\sa\sproblem\scausing\scorruption\swhen\san\sUNLOCKED\stransaction\sis\srolled\sback. +D 2015-08-20T20:25:56.029 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 4de3ef40c8b3b75c0c55ff4242a43c8ce1ad90ee F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -315,7 +315,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 388c023582b17890f10c980b30ec1922b471753b F src/os_win.c 40b3af7a47eb1107d0d69e592bec345a3b7b798a F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c fe82e368600278cfd35024c20b888e945236f675 +F src/pager.c 48a76f16a75112c9b064e21e1d1ecf0b4d87b5cd F src/pager.h 3fc23bca2e7d606ca4fbdd923713cdcdcec48297 F src/parse.y 199145cd982587d70fa3db2f9b11f7ab118f0acd F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0 @@ -1216,7 +1216,7 @@ F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unique2.test 41e7f83c6827605991160a31380148a9fc5f1339 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 F test/unlocked.test d143a871bd874311d4e5c98ce458218ca6b579fd -F test/unlocked2.test fc4c730a44c3650250467ecacca6ddb969f64405 +F test/unlocked2.test 06a7d5a366bfc6fcaf472a2411f6d45fd9255ed4 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b @@ -1368,7 +1368,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P de1ea450db33b140b11af5b801ea6a15875e774e -R db05422c8fb37e882a7e6cc4bd7c8ffb +P 3bbc31d515ba9fc920c5cbc7059d3eb1ba9c7f8e +R 9ac18b47cee85e97c9fcd4362b958c7d U dan -Z cccfd407eb244cf1cd3b5cab72e65fda +Z 032a41c4b1c95657db216d02c5116d23 diff --git a/manifest.uuid b/manifest.uuid index 7d70f3c411..ce9cb89059 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3bbc31d515ba9fc920c5cbc7059d3eb1ba9c7f8e \ No newline at end of file +7c36147846484c96023939864417b5624f3bc5f8 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index efbea0ab8e..ac2f6bf1a6 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3021,6 +3021,7 @@ static int pagerUndoCallback(void *pCtx, Pgno iPg){ static int pagerRollbackWal(Pager *pPager){ int rc; /* Return Code */ PgHdr *pList; /* List of dirty pages to revert */ + int bPage1 = 0; /* True if page 1 has been undone */ /* For all pages in the cache that are currently dirty or have already ** been written (but not committed) to the log file, do one of the @@ -3034,10 +3035,18 @@ static int pagerRollbackWal(Pager *pPager){ pList = sqlite3PcacheDirtyList(pPager->pPCache); while( pList && rc==SQLITE_OK ){ PgHdr *pNext = pList->pDirty; + if( pList->pgno==1 ) bPage1 = 1; rc = pagerUndoCallback((void *)pPager, pList->pgno); pList = pNext; } + /* If this is an UNLOCKED transaction, then page 1 must be reread from the + ** db file, even if it is not dirty. This is because the b-tree layer may + ** have already zeroed the nFree and iTrunk header fields. */ + if( rc==SQLITE_OK && bPage1==0 && pPager->pAllRead ){ + rc = pagerUndoCallback((void*)pPager, 1); + } + return rc; } diff --git a/test/unlocked2.test b/test/unlocked2.test index f808a84e7b..e9c62194a0 100644 --- a/test/unlocked2.test +++ b/test/unlocked2.test @@ -90,12 +90,72 @@ do_multiclient_test tn { sql2 { INSERT INTO t2 VALUES(randomblob(1500)); } + sql1 COMMIT + } {} + + do_test 2.$tn.3 { sql3 { PRAGMA integrity_check } } {ok} + + do_test 2.$tn.4 { + sql1 { + BEGIN UNLOCKED; + DELETE FROM t1; + } + sql2 { + DELETE FROM t2; + } + sql1 COMMIT + } {} + + do_test 2.$tn.5 { sql3 { PRAGMA integrity_check } } {ok} + + do_test 2.$tn.6 { sql1 { - COMMIT; + INSERT INTO t1 VALUES(randomblob(1500)); + INSERT INTO t1 VALUES(randomblob(1500)); + INSERT INTO t2 VALUES(randomblob(1500)); + DELETE FROM t1 WHERE rowid=1; + } + + sql1 { + BEGIN UNLOCKED; + DELETE FROM t1 WHERE rowid=2; + } + + sql2 { + DELETE FROM t2; } + + sql1 COMMIT } {} - do_test 2.$tn.3 { sql3 { PRAGMA integrity_check } } {ok} + do_test 2.$tn.7 { sql3 { PRAGMA integrity_check } } {ok} } +reset_db +do_execsql_test 3.0 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES(randomblob(1500), randomblob(1500)); + DELETE FROM t1; +} {wal} + +db close +sqlite3 db test.db +breakpoint + +do_execsql_test 3.1 { + BEGIN UNLOCKED; + INSERT INTO t1 VALUES(1, 2); + ROLLBACK; +} + +do_execsql_test 3.2 { + PRAGMA integrity_check; +} {ok} + +do_execsql_test 3.3 { + PRAGMA freelist_count; +} {2} + finish_test + From 64b310ed1aae9043983c81f7175d984024ce1717 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 21 Aug 2015 14:21:22 +0000 Subject: [PATCH 007/179] Add extra tests and a fix for rollbacks of UNLOCKED transactions. FossilOrigin-Name: 82cd837e72ed4cf5821be717369694be29a2997e --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/wal.c | 21 ++++++++++++++++++--- test/unlocked2.test | 46 +++++++++++++++++++++++++++++++++++++-------- 4 files changed, 64 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index ba7247bf36..dd80228576 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\scausing\scorruption\swhen\san\sUNLOCKED\stransaction\sis\srolled\sback. -D 2015-08-20T20:25:56.029 +C Add\sextra\stests\sand\sa\sfix\sfor\srollbacks\sof\sUNLOCKED\stransactions. +D 2015-08-21T14:21:22.365 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 4de3ef40c8b3b75c0c55ff4242a43c8ce1ad90ee F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -402,7 +402,7 @@ F src/vdbesort.c f5009e7a35e3065635d8918b9a31f498a499976b F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 F src/vtab.c 082b35a25a26e3d36f365ca8cd73c1922532f05e F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb -F src/wal.c 8bb1130dc1876e1c1c3fa082a5b0ec6437feb8e8 +F src/wal.c 37b25bbb52d6a736397d11e85068b1a496970fb2 F src/wal.h 5aaed8ca6cad1406088042ff8767951a6a3c7b56 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c 909eba3b6db984eb2adfbca9de2c237ee7056adb @@ -1216,7 +1216,7 @@ F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unique2.test 41e7f83c6827605991160a31380148a9fc5f1339 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 F test/unlocked.test d143a871bd874311d4e5c98ce458218ca6b579fd -F test/unlocked2.test 06a7d5a366bfc6fcaf472a2411f6d45fd9255ed4 +F test/unlocked2.test aca373d7639fd11c946a8e1a78c57bab0f18bc8e F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b @@ -1368,7 +1368,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3bbc31d515ba9fc920c5cbc7059d3eb1ba9c7f8e -R 9ac18b47cee85e97c9fcd4362b958c7d +P 7c36147846484c96023939864417b5624f3bc5f8 +R d431131c35d67ee146c27b397321fd1c U dan -Z 032a41c4b1c95657db216d02c5116d23 +Z 6bf329345f5b33de62ef6fd5bfd9b35f diff --git a/manifest.uuid b/manifest.uuid index ce9cb89059..f765222789 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7c36147846484c96023939864417b5624f3bc5f8 \ No newline at end of file +82cd837e72ed4cf5821be717369694be29a2997e \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 809ce2b109..ac7e522ff5 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2635,10 +2635,25 @@ int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPage1, Bitvec *pAllRead){ ); rc = SQLITE_BUSY_SNAPSHOT; }else if( (pPg = sqlite3PagerLookup(pPager, aPgno[i])) ){ - if( sqlite3PagerIswriteable(pPg)==0 ){ - sqlite3PcacheDrop(pPg); - }else{ + /* Page aPgno[i], which is present in the pager cache, has been + ** modified since the current UNLOCKED transaction was started. + ** However it was not read by the current transaction, so is not + ** a conflict. There are two possibilities: (a) the page was + ** allocated at the of the file by the current transaction or + ** (b) was present in the cache at the start of the transaction. + ** + ** For case (a), do nothing. This page will be moved within the + ** database file by the commit code to avoid the conflict. The + ** call to PagerUnref() is to release the reference grabbed by + ** the sqlite3PagerLookup() above. + ** + ** In case (b), drop the page from the cache - otherwise + ** following the snapshot upgrade the cache would be inconsistent + ** with the database as stored on disk. */ + if( sqlite3PagerIswriteable(pPg) ){ sqlite3PagerUnref(pPg); + }else{ + sqlite3PcacheDrop(pPg); } } } diff --git a/test/unlocked2.test b/test/unlocked2.test index e9c62194a0..1ec9050b60 100644 --- a/test/unlocked2.test +++ b/test/unlocked2.test @@ -131,6 +131,18 @@ do_multiclient_test tn { do_test 2.$tn.7 { sql3 { PRAGMA integrity_check } } {ok} } +#------------------------------------------------------------------------- +# When an UNLOCKED transaction is opened on a database, the nFree and +# iTrunk header fields of the cached version of page 1 are both set +# to 0. This allows an UNLOCKED transaction to use its own private +# free-page-list, which is merged with the main database free-list when +# the transaction is committed. +# +# The following tests check that nFree/iTrunk are correctly restored if +# an UNLOCKED transaction is rolled back, and that savepoint rollbacks +# that occur within UNLOCKED transactions do not incorrectly restore +# these fields to their on-disk values. +# reset_db do_execsql_test 3.0 { PRAGMA journal_mode = wal; @@ -139,23 +151,41 @@ do_execsql_test 3.0 { DELETE FROM t1; } {wal} -db close -sqlite3 db test.db -breakpoint - do_execsql_test 3.1 { BEGIN UNLOCKED; INSERT INTO t1 VALUES(1, 2); ROLLBACK; } -do_execsql_test 3.2 { - PRAGMA integrity_check; -} {ok} +do_execsql_test 3.2 { PRAGMA integrity_check } {ok} +do_execsql_test 3.3 { PRAGMA freelist_count } {2} -do_execsql_test 3.3 { +do_execsql_test 3.4.1 { + BEGIN UNLOCKED; + PRAGMA freelist_count; +} {2} +do_execsql_test 3.4.2 { + SAVEPOINT xyz; + INSERT INTO t1 VALUES(randomblob(1500), NULL); + PRAGMA freelist_count; +} {0} +do_execsql_test 3.4.3 { + ROLLBACK TO xyz; +} {} +do_execsql_test 3.4.4 { PRAGMA freelist_count } {0} +do_execsql_test 3.4.5 { COMMIT; PRAGMA freelist_count } {2} +do_execsql_test 3.4.6 { PRAGMA integrity_check } {ok} + +do_execsql_test 3.5.1 { + BEGIN UNLOCKED; + UPDATE t1 SET x=randomblob(10) WHERE y=555; + PRAGMA freelist_count; +} {0} +do_execsql_test 3.5.2 { + ROLLBACK; PRAGMA freelist_count; } {2} +do_execsql_test 3.5.3 { PRAGMA integrity_check } {ok} finish_test From 70af25d03de9e183ee914cceedbdfbe5feb7bd7c Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 21 Aug 2015 17:57:16 +0000 Subject: [PATCH 008/179] Fix a problem with UNLOCKED transactions that free pages allocated within the same transaction. FossilOrigin-Name: 227bb8a1815c4dc6084970f06b0a6bfccdff3fd2 --- manifest | 14 +++++----- manifest.uuid | 2 +- src/btree.c | 62 ++++++++++++++++++++++++++++----------------- test/unlocked2.test | 34 +++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 31 deletions(-) diff --git a/manifest b/manifest index f5c3e2811a..ed83b7c162 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\swith\sthis\sbranch. -D 2015-08-21T16:22:04.553 +C Fix\sa\sproblem\swith\sUNLOCKED\stransactions\sthat\sfree\spages\sallocated\swithin\sthe\ssame\stransaction. +D 2015-08-21T17:57:16.096 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 4f663b6b4954b9b1eb0e6f08387688a93b57542d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -278,7 +278,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 4d9134dc988a87838c06056c89c0e8c4700a0452 F src/bitvec.c d1f21d7d91690747881f03940584f4cc548c9d3d F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 -F src/btree.c a00a7c4809d642bf6c63979e73face9fed53dc66 +F src/btree.c fca7884ea849190eff530f358f96c7304fcd2595 F src/btree.h 00d4cdb747c4172a5566faf037116985dbbc377e F src/btreeInt.h 1f7258e1f35ba3cc2197f9019f523dd20d59a0d8 F src/build.c e47b6fffe14a28d9050e6747beebb01597d37542 @@ -1226,7 +1226,7 @@ F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unique2.test 41e7f83c6827605991160a31380148a9fc5f1339 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 F test/unlocked.test d143a871bd874311d4e5c98ce458218ca6b579fd -F test/unlocked2.test aca373d7639fd11c946a8e1a78c57bab0f18bc8e +F test/unlocked2.test 4861218039ca2481613d7e6a999226185222c93a F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b @@ -1378,7 +1378,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 82cd837e72ed4cf5821be717369694be29a2997e 7b8d17dd840f64dac9a018a4547a97de799e94ab -R a6aada1e15ca09028aeff07fdb262305 +P deaf3b18569025d8d168da29fae76142cfffe9e5 +R f09b37f1aebb9ccbb19d5d48d89374eb U dan -Z 957a88b73b652ebea177f80c25ee0401 +Z 1ea7f2e6e188cedc2dae0825d7ceb725 diff --git a/manifest.uuid b/manifest.uuid index ee7b5e2772..4ee4bbbd4b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -deaf3b18569025d8d168da29fae76142cfffe9e5 \ No newline at end of file +227bb8a1815c4dc6084970f06b0a6bfccdff3fd2 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index cc97fd8623..0bd34c7439 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3870,7 +3870,7 @@ static int btreeFixUnlocked(Btree *p){ if( rc==SQLITE_OK ){ Pgno nHPage = get4byte(&p1[28]); - Pgno nFinal = nHPage; + Pgno nFinal = nHPage; /* Size of db after transaction merge */ if( sqlite3PagerIswriteable(pPage1->pDbPage) ){ Pgno iHTrunk = get4byte(&p1[32]); @@ -3892,6 +3892,9 @@ static int btreeFixUnlocked(Btree *p){ } if( nHPageiFirst through @@ -3902,22 +3905,28 @@ static int btreeFixUnlocked(Btree *p){ Pgno iLast = MIN(nPage, nHPage); /* Last page to move */ Pgno iPg; - nFinal = MAX(nPage, nHPage); /* Final size of database */ + nFinal = MAX(nPage, nHPage); for(iPg=pMap->iFirst; iPg<=iLast && rc==SQLITE_OK; iPg++){ MemPage *pPg = 0; Pgno iNew; /* New page number for pPg */ PtrmapEntry *pEntry; /* Pointer map entry for page iPg */ - btreeGetPage(pBt, iPg, &pPg, 0); - assert( sqlite3PagerIswriteable(pPg->pDbPage) ); - assert( sqlite3PagerPageRefcount(pPg->pDbPage)==1 ); pEntry = &pMap->aPtr[iPg - pMap->iFirst]; - iNew = ++nFinal; - - rc = relocatePage(pBt, pPg, pEntry->eType, pEntry->parent, iNew, 1); - releasePageNotNull(pPg); + if( pEntry->eType==PTRMAP_FREEPAGE ){ + MemPage *pFree = 0; + Pgno dummy; + rc = allocateBtreePage(pBt, &pFree, &dummy, iPg, BTALLOC_EXACT); + releasePage(pFree); + assert( rc!=SQLITE_OK || dummy==iPg ); + }else{ + btreeGetPage(pBt, iPg, &pPg, 0); + assert( sqlite3PagerIswriteable(pPg->pDbPage) ); + assert( sqlite3PagerPageRefcount(pPg->pDbPage)==1 ); + iNew = ++nFinal; + rc = relocatePage(pBt, pPg, pEntry->eType, pEntry->parent, iNew, 1); + releasePageNotNull(pPg); + } } - put4byte(&p1[28], nFinal); } } @@ -5775,14 +5784,16 @@ static int allocateBtreePage( Pgno mxPage; /* Total size of the database file */ assert( sqlite3_mutex_held(pBt->mutex) ); - assert( eMode==BTALLOC_ANY || (nearby>0 && IfNotOmitAV(pBt->autoVacuum)) ); + assert( eMode==BTALLOC_ANY || (nearby>0 && ISAUTOVACUUM) + || (eMode==BTALLOC_EXACT && sqlite3PagerIsUnlocked(pBt->pPager)) + ); pPage1 = pBt->pPage1; mxPage = btreePagecount(pBt); /* EVIDENCE-OF: R-05119-02637 The 4-byte big-endian integer at offset 36 ** stores stores the total number of pages on the freelist. */ n = get4byte(&pPage1->aData[36]); testcase( n==mxPage-1 ); - if( n>=mxPage ){ + if( sqlite3PagerIsUnlocked(pBt->pPager)==0 && n>=mxPage ){ return SQLITE_CORRUPT_BKPT; } @@ -5791,7 +5802,6 @@ static int allocateBtreePage( ** of these operations involve modifying page 1 header fields, page 1 ** will definitely be written by this transaction. If this is an UNLOCKED ** transaction, ensure the BtreePtrmap structure has been allocated. */ - assert( eMode!=BTALLOC_EXACT || sqlite3PagerIsUnlocked(pBt->pPager)==0 ); rc = sqlite3PagerWrite(pPage1->pDbPage); if( rc ) return rc; @@ -5805,19 +5815,25 @@ static int allocateBtreePage( ** shows that the page 'nearby' is somewhere on the free-list, then ** the entire-list will be searched for that page. */ -#ifndef SQLITE_OMIT_AUTOVACUUM if( eMode==BTALLOC_EXACT ){ - if( nearby<=mxPage ){ - u8 eType; - assert( nearby>0 ); - assert( pBt->autoVacuum ); - rc = ptrmapGet(pBt, nearby, &eType, 0); - if( rc ) return rc; - if( eType==PTRMAP_FREEPAGE ){ - searchList = 1; + assert( ISAUTOVACUUM==!sqlite3PagerIsUnlocked(pBt->pPager) ); + if( ISAUTOVACUUM ){ + if( nearby<=mxPage ){ + u8 eType; + assert( nearby>0 ); + assert( pBt->autoVacuum ); + rc = ptrmapGet(pBt, nearby, &eType, 0); + if( rc ) return rc; + if( eType==PTRMAP_FREEPAGE ){ + searchList = 1; + } } + }else{ + searchList = 1; } - }else if( eMode==BTALLOC_LE ){ + } +#ifndef SQLITE_OMIT_AUTOVACUUM + else if( eMode==BTALLOC_LE ){ searchList = 1; } #endif diff --git a/test/unlocked2.test b/test/unlocked2.test index 1ec9050b60..42742111c8 100644 --- a/test/unlocked2.test +++ b/test/unlocked2.test @@ -187,5 +187,39 @@ do_execsql_test 3.5.2 { } {2} do_execsql_test 3.5.3 { PRAGMA integrity_check } {ok} +#------------------------------------------------------------------------- +# Test that nothing goes wrong if an UNLOCKED transaction allocates a +# page at the end of the file, frees it within the same transaction, and +# then has to move the same page to avoid a conflict on COMMIT. +# +do_multiclient_test tn { + do_test 4.$tn.1 { + sql1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x); + CREATE TABLE t2(x); + } + } {wal} + + do_test 4.$tn.2 { + sql1 { + BEGIN UNLOCKED; + INSERT INTO t1 VALUES(randomblob(1500)); + INSERT INTO t1 VALUES(randomblob(1500)); + DELETE FROM t1 WHERE rowid = 1; + } + + sql2 { + INSERT INTO t2 VALUES(randomblob(1500)); + INSERT INTO t2 VALUES(randomblob(1500)); + INSERT INTO t2 VALUES(randomblob(1500)); + INSERT INTO t2 VALUES(randomblob(1500)); + DELETE FROM t2 WHERE rowid IN (1, 2); + } + + sql1 COMMIT + } {} +} + finish_test From 572a21c8e99373636abb5a76f019794bcc57c7a9 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 21 Aug 2015 18:55:22 +0000 Subject: [PATCH 009/179] When committing an UNLOCKED transaction, try to move pages allocated at the end of the file to free slots within the file (like an incremental-vacuum operation does). FossilOrigin-Name: 069679162d8d50e9731831e658aa58f280dbb3e7 --- manifest | 14 ++++++------- manifest.uuid | 2 +- src/btree.c | 49 ++++++++++++++++++++++++++++++++------------- test/unlocked2.test | 4 ++++ 4 files changed, 47 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index ed83b7c162..ce414f1bbb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sUNLOCKED\stransactions\sthat\sfree\spages\sallocated\swithin\sthe\ssame\stransaction. -D 2015-08-21T17:57:16.096 +C When\scommitting\san\sUNLOCKED\stransaction,\stry\sto\smove\spages\sallocated\sat\sthe\send\sof\sthe\sfile\sto\sfree\sslots\swithin\sthe\sfile\s(like\san\sincremental-vacuum\soperation\sdoes). +D 2015-08-21T18:55:22.935 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 4f663b6b4954b9b1eb0e6f08387688a93b57542d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -278,7 +278,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 4d9134dc988a87838c06056c89c0e8c4700a0452 F src/bitvec.c d1f21d7d91690747881f03940584f4cc548c9d3d F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 -F src/btree.c fca7884ea849190eff530f358f96c7304fcd2595 +F src/btree.c e455c3dbe8534aae357eab8ede1e280b6e838af0 F src/btree.h 00d4cdb747c4172a5566faf037116985dbbc377e F src/btreeInt.h 1f7258e1f35ba3cc2197f9019f523dd20d59a0d8 F src/build.c e47b6fffe14a28d9050e6747beebb01597d37542 @@ -1226,7 +1226,7 @@ F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unique2.test 41e7f83c6827605991160a31380148a9fc5f1339 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 F test/unlocked.test d143a871bd874311d4e5c98ce458218ca6b579fd -F test/unlocked2.test 4861218039ca2481613d7e6a999226185222c93a +F test/unlocked2.test 1e645e6f6808e4303f1766d8c615c22714f31541 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b @@ -1378,7 +1378,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P deaf3b18569025d8d168da29fae76142cfffe9e5 -R f09b37f1aebb9ccbb19d5d48d89374eb +P 227bb8a1815c4dc6084970f06b0a6bfccdff3fd2 +R 59f0ca349927e5f594c9b22ba4f42852 U dan -Z 1ea7f2e6e188cedc2dae0825d7ceb725 +Z 06941d0e9b57ce8c4c11e26554097e38 diff --git a/manifest.uuid b/manifest.uuid index 4ee4bbbd4b..167dab9ea2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -227bb8a1815c4dc6084970f06b0a6bfccdff3fd2 \ No newline at end of file +069679162d8d50e9731831e658aa58f280dbb3e7 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 0bd34c7439..bf76254afe 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3860,7 +3860,6 @@ static int btreeFixUnlocked(Btree *p){ BtreePtrmap *pMap = pBt->pMap; Pgno iTrunk = get4byte(&p1[32]); Pgno nPage = btreePagecount(pBt); - Pgno nOrig = pMap->iFirst-1; u32 nFree = get4byte(&p1[36]); assert( sqlite3PagerIsUnlocked(pPager) ); @@ -3891,10 +3890,12 @@ static int btreeFixUnlocked(Btree *p){ }; } - if( nHPageiFirst-1) ){ + /* The database consisted of (pMap->iFirst-1) pages when the current + ** unlocked transaction was opened. And an unlocked transaction may + ** not be executed on an auto-vacuum database - so the db should + ** not have shrunk since the transaction was opened. Therefore nHPage + ** should be set to (pMap->iFirst-1) or greater. */ rc = SQLITE_CORRUPT_BKPT; }else{ /* The current transaction allocated pages pMap->iFirst through @@ -3904,8 +3905,9 @@ static int btreeFixUnlocked(Btree *p){ */ Pgno iLast = MIN(nPage, nHPage); /* Last page to move */ Pgno iPg; + Pgno nCurrent; /* Current size of db */ - nFinal = MAX(nPage, nHPage); + nCurrent = MAX(nPage, nHPage); for(iPg=pMap->iFirst; iPg<=iLast && rc==SQLITE_OK; iPg++){ MemPage *pPg = 0; Pgno iNew; /* New page number for pPg */ @@ -3922,11 +3924,35 @@ static int btreeFixUnlocked(Btree *p){ btreeGetPage(pBt, iPg, &pPg, 0); assert( sqlite3PagerIswriteable(pPg->pDbPage) ); assert( sqlite3PagerPageRefcount(pPg->pDbPage)==1 ); - iNew = ++nFinal; + iNew = ++nCurrent; rc = relocatePage(pBt, pPg, pEntry->eType, pEntry->parent, iNew, 1); releasePageNotNull(pPg); } } + sqlite3PagerSetDbsize(pPager, nCurrent); + + nFree = get4byte(&p1[36]); + nFinal = MAX(nCurrent-nFree, nHPage); + + for(iPg=nFinal+1; rc==SQLITE_OK && iPgaPtr[iPg - pMap->iFirst]; + if( pEntry->eType==PTRMAP_FREEPAGE ){ + rc = allocateBtreePage(pBt, &pFree, &iNew, iPg, BTALLOC_EXACT); + releasePage(pFree); + }else{ + rc = allocateBtreePage(pBt, &pFree, &iNew, nFinal, BTALLOC_LE); + releasePage(pFree); + if( rc==SQLITE_OK ){ + MemPage *pPg = 0; + btreeGetPage(pBt, iPg, &pPg, 0); + rc = relocatePage(pBt, pPg, pEntry->eType, pEntry->parent,iNew,1); + } + } + } put4byte(&p1[28], nFinal); } } @@ -5784,9 +5810,7 @@ static int allocateBtreePage( Pgno mxPage; /* Total size of the database file */ assert( sqlite3_mutex_held(pBt->mutex) ); - assert( eMode==BTALLOC_ANY || (nearby>0 && ISAUTOVACUUM) - || (eMode==BTALLOC_EXACT && sqlite3PagerIsUnlocked(pBt->pPager)) - ); + assert( eMode==BTALLOC_ANY || (nearby>0 && REQUIRE_PTRMAP ) ); pPage1 = pBt->pPage1; mxPage = btreePagecount(pBt); /* EVIDENCE-OF: R-05119-02637 The 4-byte big-endian integer at offset 36 @@ -5831,12 +5855,9 @@ static int allocateBtreePage( }else{ searchList = 1; } - } -#ifndef SQLITE_OMIT_AUTOVACUUM - else if( eMode==BTALLOC_LE ){ + }else if( eMode==BTALLOC_LE ){ searchList = 1; } -#endif /* Decrement the free-list count by 1. Set iTrunk to the index of the ** first free-list trunk page. iPrevTrunk is initially 1. diff --git a/test/unlocked2.test b/test/unlocked2.test index 42742111c8..e5b102aa02 100644 --- a/test/unlocked2.test +++ b/test/unlocked2.test @@ -25,6 +25,7 @@ do_multiclient_test tn { CREATE TABLE t2(y); } } {wal} + do_test 1.$tn.5 { sql3 { PRAGMA integrity_check } } {ok} # Test that an UNLOCKED transaction that allocates/frees no pages does # not conflict with a transaction that does allocate pages. @@ -40,6 +41,7 @@ do_multiclient_test tn { COMMIT; } } {} + do_test 1.$tn.5 { sql3 { PRAGMA integrity_check } } {ok} # But that an UNLOCKED transaction does conflict with a transaction # that modifies the db schema. @@ -53,6 +55,7 @@ do_multiclient_test tn { } list [catch { sql1 COMMIT } msg] $msg } {1 {database is locked}} + do_test 1.$tn.5 { sql3 { PRAGMA integrity_check } } {ok} # Test that an UNLOCKED transaction that allocates at least one page # does not conflict with a transaction that allocates no pages. @@ -65,6 +68,7 @@ do_multiclient_test tn { sql2 { INSERT INTO t2 VALUES(8); } + breakpoint sql1 { COMMIT; } From b87b25f21926b2fad83fa28fa0bc2827ef170a1a Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 21 Aug 2015 20:11:23 +0000 Subject: [PATCH 010/179] Fix many minor issues in the unlocked transaction code. FossilOrigin-Name: 53aaeea6c98f82f3b55d4b3ab139ee41b727243c --- manifest | 26 +++++++++++++------------- manifest.uuid | 2 +- src/btree.c | 2 +- src/main.c | 1 - src/pager.c | 5 +---- src/pager.h | 5 ++--- src/vacuum.c | 2 +- src/vdbeaux.c | 2 +- src/wal.c | 19 ------------------- src/wal.h | 2 -- 10 files changed, 20 insertions(+), 46 deletions(-) diff --git a/manifest b/manifest index ce414f1bbb..4c65a4a62a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\scommitting\san\sUNLOCKED\stransaction,\stry\sto\smove\spages\sallocated\sat\sthe\send\sof\sthe\sfile\sto\sfree\sslots\swithin\sthe\sfile\s(like\san\sincremental-vacuum\soperation\sdoes). -D 2015-08-21T18:55:22.935 +C Fix\smany\sminor\sissues\sin\sthe\sunlocked\stransaction\scode. +D 2015-08-21T20:11:23.106 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 4f663b6b4954b9b1eb0e6f08387688a93b57542d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -278,7 +278,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 4d9134dc988a87838c06056c89c0e8c4700a0452 F src/bitvec.c d1f21d7d91690747881f03940584f4cc548c9d3d F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 -F src/btree.c e455c3dbe8534aae357eab8ede1e280b6e838af0 +F src/btree.c dc80ac8374105e3caf298a6533e3312101691330 F src/btree.h 00d4cdb747c4172a5566faf037116985dbbc377e F src/btreeInt.h 1f7258e1f35ba3cc2197f9019f523dd20d59a0d8 F src/build.c e47b6fffe14a28d9050e6747beebb01597d37542 @@ -301,7 +301,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 92bafa308607dd985ca389a788cd9e0a2b608712 F src/loadext.c dfcee8c7c032cd0fd55af3e0fc1fcfb01e426df2 -F src/main.c f4a8a75649073b85048505e2bdda4419936d9eb6 +F src/main.c e17fcffae4306a9b8334faf3bac80d7396850b54 F src/malloc.c 19461e159bccf0e2cf06a50e867963d0a7b124a8 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c abe6ee469b6c5a35c7f22bfeb9c9bac664a1c987 @@ -323,8 +323,8 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 388c023582b17890f10c980b30ec1922b471753b F src/os_win.c 40b3af7a47eb1107d0d69e592bec345a3b7b798a F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 48a76f16a75112c9b064e21e1d1ecf0b4d87b5cd -F src/pager.h 3fc23bca2e7d606ca4fbdd923713cdcdcec48297 +F src/pager.c 951b7bae30fcbc2a2d380fa68f8bc68286ae0760 +F src/pager.h 1e7b0fc3846b71bd95b4b3300820d756895cb4ef F src/parse.y e9accdb2cb1795f75f478e7ce89e17b19d7d4da7 F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0 F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9 @@ -398,20 +398,20 @@ F src/trigger.c 322f23aad694e8f31d384dcfa386d52a48d3c52f F src/update.c 487747b328b7216bb7f6af0695d6937d5c9e605f F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c bc9dd64b5db544218b871b66243871c202b2781f -F src/vacuum.c 4e7e18d889662eda281855a7125e56269069f215 +F src/vacuum.c d35c7291e94a470ee47695c22559a69aef9fdda1 F src/vdbe.c 97b07a1af65971dab3d326742e912ac8c12108dd F src/vdbe.h 7a75045d879118b9d3af7e8b3c108f2f27c51473 F src/vdbeInt.h 8b54e01ad0463590e7cffabce0bc36da9ee4f816 F src/vdbeapi.c bda74ef4b5103d7b4a4be36f936d3cf2b56a7d6f -F src/vdbeaux.c 0d62c6c2522d4a04e051d3862dfe7ae2f5c95598 +F src/vdbeaux.c 4fae90f55736b7537d0fcd2afe52849edbbb7ecd F src/vdbeblob.c 4f2e8e075d238392df98c5e03a64342465b03f90 F src/vdbemem.c ae38a0d35ae71cf604381a887c170466ba518090 F src/vdbesort.c f5009e7a35e3065635d8918b9a31f498a499976b F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 F src/vtab.c d31174e4c8f592febab3fa7f69e18320b4fd657a F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb -F src/wal.c 37b25bbb52d6a736397d11e85068b1a496970fb2 -F src/wal.h 5aaed8ca6cad1406088042ff8767951a6a3c7b56 +F src/wal.c 0ba4899f02457be91fe5bc734638fee00c807f64 +F src/wal.h a701096b9bf5761b8c5f4b07082249e1950afc9a F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba F src/where.c 66518a14a1238611aa0744d6980b6b7f544f4816 F src/whereInt.h 880a8599226ac1c00203490d934f3ed79b292572 @@ -1378,7 +1378,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 227bb8a1815c4dc6084970f06b0a6bfccdff3fd2 -R 59f0ca349927e5f594c9b22ba4f42852 +P 069679162d8d50e9731831e658aa58f280dbb3e7 +R 98654e6b793f843bd7dce43cd408dae9 U dan -Z 06941d0e9b57ce8c4c11e26554097e38 +Z 8f0a71142a3eb918cc8670c51bd39224 diff --git a/manifest.uuid b/manifest.uuid index 167dab9ea2..c4520623e3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -069679162d8d50e9731831e658aa58f280dbb3e7 \ No newline at end of file +53aaeea6c98f82f3b55d4b3ab139ee41b727243c \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index bf76254afe..fd7afc073c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3906,8 +3906,8 @@ static int btreeFixUnlocked(Btree *p){ Pgno iLast = MIN(nPage, nHPage); /* Last page to move */ Pgno iPg; Pgno nCurrent; /* Current size of db */ - nCurrent = MAX(nPage, nHPage); + for(iPg=pMap->iFirst; iPg<=iLast && rc==SQLITE_OK; iPg++){ MemPage *pPg = 0; Pgno iNew; /* New page number for pPg */ diff --git a/src/main.c b/src/main.c index fdf1584933..575cad92c5 100644 --- a/src/main.c +++ b/src/main.c @@ -2744,7 +2744,6 @@ static int openDatabase( memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = SQLITE_DEFAULT_WORKER_THREADS; db->autoCommit = 1; - db->bUnlocked = 0; db->nextAutovac = -1; db->szMmap = sqlite3GlobalConfig.szMmap; db->nextPagesize = 0; diff --git a/src/pager.c b/src/pager.c index ac2f6bf1a6..c55db40150 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5631,8 +5631,6 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ pPager->pAllRead = sqlite3BitvecCreate(pPager->dbSize); if( pPager->pAllRead==0 ){ rc = SQLITE_NOMEM; - }else{ - rc = sqlite3BitvecSet(pPager->pAllRead, 1); } } }else{ @@ -6155,8 +6153,7 @@ void sqlite3PagerDropExclusiveLock(Pager *pPager){ } /* -** Return true if this is a WAL database and snapshot upgrade is required -** before the current transaction can be committed. +** Return true if this pager is currently within an UNLOCKED transaction. */ int sqlite3PagerIsUnlocked(Pager *pPager){ return pPager->pAllRead!=0; diff --git a/src/pager.h b/src/pager.h index 92e5546282..c097c97c5b 100644 --- a/src/pager.h +++ b/src/pager.h @@ -145,7 +145,6 @@ int sqlite3PagerMovepage(Pager*,DbPage*,Pgno,int); int sqlite3PagerPageRefcount(DbPage*); void *sqlite3PagerGetData(DbPage *); void *sqlite3PagerGetExtra(DbPage *); -int sqlite3PagerIsDirty(DbPage*); /* Functions used to manage pager transactions and savepoints. */ void sqlite3PagerPagecount(Pager*, int*); @@ -159,8 +158,6 @@ int sqlite3PagerOpenSavepoint(Pager *pPager, int n); int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint); int sqlite3PagerSharedLock(Pager *pPager); -void sqlite3PagerDropExclusiveLock(Pager*); -int sqlite3PagerIsUnlocked(Pager*); #ifndef SQLITE_OMIT_WAL int sqlite3PagerCheckpoint(Pager *pPager, int, int*, int*); @@ -197,6 +194,8 @@ void sqlite3PagerTruncateImage(Pager*,Pgno); void sqlite3PagerRekey(DbPage*, Pgno, u16); +void sqlite3PagerDropExclusiveLock(Pager*); +int sqlite3PagerIsUnlocked(Pager*); int sqlite3PagerIswriteable(DbPage*); int sqlite3PagerUpgradeSnapshot(Pager *pPager, DbPage*); void sqlite3PagerSetDbsize(Pager *pPager, Pgno); diff --git a/src/vacuum.c b/src/vacuum.c index 10f14cbf47..46f2249ebb 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -356,7 +356,7 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ ** is closed by the DETACH. */ db->autoCommit = 1; - db->bUnlocked = 0; + assert( db->bUnlocked==0 ); if( pDb ){ sqlite3BtreeClose(pDb->pBt); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index ce48707be7..354deb7294 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2571,7 +2571,7 @@ int sqlite3VdbeHalt(Vdbe *p){ } assert( db->nVdbeActive>0 || db->autoCommit==0 || db->nStatement==0 ); - return ((p->rc & 0xFF)==SQLITE_BUSY ? SQLITE_BUSY : SQLITE_OK); + return (p->rc==SQLITE_BUSY ? SQLITE_BUSY : SQLITE_OK); } diff --git a/src/wal.c b/src/wal.c index ac7e522ff5..a9d8a44086 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2671,25 +2671,6 @@ void sqlite3WalUpgradeSnapshot(Wal *pWal){ memcpy(&pWal->hdr, (void*)walIndexHdr(pWal), sizeof(WalIndexHdr)); } -/* -** This function is only ever called while committing an UNLOCKED -** transaction, after the caller has already obtained the WRITER lock -** (by calling the sqlite3WalLockForCommit() routine). This function -** returns true if the transaction was prepared against a database -** snapshot older than the current head of the wal file. -** -** Note that this will only work as described if the database is -** currently executing an UNLOCKED transaction, as it assumes that -** pWal->hdr has not been modified since the beginning of the -** transaction. This may not be true for a non-UNLOCKED transaction, -** as pWal->hdr is updated if any pages are spilled to the wal file -** while the transaction is executing. -*/ -int sqlite3WalCommitRequiresUpgrade(Wal *pWal){ - assert( pWal->writeLock ); - return memcmp(&pWal->hdr, (void*)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0; -} - /* ** End a write transaction. The commit has already been done. This ** routine merely releases the lock. diff --git a/src/wal.h b/src/wal.h index 5b4ca15943..0609eded27 100644 --- a/src/wal.h +++ b/src/wal.h @@ -126,9 +126,7 @@ int sqlite3WalExclusiveMode(Wal *pWal, int op); */ int sqlite3WalHeapMemory(Wal *pWal); -/* Return true if the WRITER lock is held. False otherwise. */ int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPg, Bitvec *pRead); -int sqlite3WalCommitRequiresUpgrade(Wal *pWal); void sqlite3WalUpgradeSnapshot(Wal *pWal); #ifdef SQLITE_ENABLE_ZIPVFS From f5cebf71fe65cab9dc4fd5b9ce9b96467e3d283f Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 22 Aug 2015 17:28:55 +0000 Subject: [PATCH 011/179] Add further tests for deferred page allocation. And fixes for the same. FossilOrigin-Name: ed0a31be726e60115a5dd73d4ed580201b400ab7 --- manifest | 17 +-- manifest.uuid | 2 +- src/btree.c | 282 ++++++++++++++++++++++++-------------------- src/btreeInt.h | 5 +- test/unlocked2.test | 34 +++++- test/unlocked3.test | 110 +++++++++++++++++ 6 files changed, 312 insertions(+), 138 deletions(-) create mode 100644 test/unlocked3.test diff --git a/manifest b/manifest index 6840db861c..bc04296ba0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sfurther\strunk\schanges. -D 2015-08-22T07:56:49.673 +C Add\sfurther\stests\sfor\sdeferred\spage\sallocation.\sAnd\sfixes\sfor\sthe\ssame. +D 2015-08-22T17:28:55.533 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -279,9 +279,9 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 4d9134dc988a87838c06056c89c0e8c4700a0452 F src/bitvec.c d1f21d7d91690747881f03940584f4cc548c9d3d F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 -F src/btree.c dc80ac8374105e3caf298a6533e3312101691330 +F src/btree.c 245bc696824d5107f3570905325c6ac5b05c18f8 F src/btree.h 00d4cdb747c4172a5566faf037116985dbbc377e -F src/btreeInt.h 1f7258e1f35ba3cc2197f9019f523dd20d59a0d8 +F src/btreeInt.h 0c19847f87ab82e4f5e67750a069f10829475da6 F src/build.c e47b6fffe14a28d9050e6747beebb01597d37542 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f @@ -1228,7 +1228,8 @@ F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unique2.test 41e7f83c6827605991160a31380148a9fc5f1339 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 F test/unlocked.test d143a871bd874311d4e5c98ce458218ca6b579fd -F test/unlocked2.test 1e645e6f6808e4303f1766d8c615c22714f31541 +F test/unlocked2.test 7f913b404cda03f22bc7b299e8eb82a34d2880c5 +F test/unlocked3.test 4facd06286885faa9bd53713fa3d701ca403aeb2 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b @@ -1380,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9021f7369f4b32db6126b3dc0ac75d11039625be 213a6c5ccbcfe4495d45e3608e99a6398751aeed -R d0dd8783a44e8da0a2326d10e188c817 +P c2327a3b8e5d604ab948b1e9f6cfc401429e51db +R c62a3f455b34280d099a1332c1c4f1e4 U dan -Z dd4c495d35a4ec7386b2f6d290b0dddd +Z 1442e612d3eba52580ab50755ceb943f diff --git a/manifest.uuid b/manifest.uuid index 4a13cf0ec4..545070a432 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c2327a3b8e5d604ab948b1e9f6cfc401429e51db \ No newline at end of file +ed0a31be726e60115a5dd73d4ed580201b400ab7 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index fd7afc073c..53b769720d 100644 --- a/src/btree.c +++ b/src/btree.c @@ -439,19 +439,14 @@ static void downgradeAllSharedCacheTableLocks(Btree *p){ #endif /* SQLITE_OMIT_SHARED_CACHE */ + +#ifdef SQLITE_ENABLE_UNLOCKED /* -** The following structure stores the in-memory pointer map used for newly -** allocated pages in UNLOCKED transactions. Such pages are always allocated -** in a contiguous block (from the end of the file) starting with page -** BtreePtrmap.iFirst. -** -** The page number for the parent page iFirst is stored in aPtr[0]. For -** (iFirst+1), aPtr[1]. A zero value indicates that the page has not -** been allocated. -** -** +** The following structure - BtreePtrmap - stores the in-memory pointer map +** used for newly allocated pages in UNLOCKED transactions. Such pages are +** always allocated in a contiguous block (from the end of the file) starting +** with page BtreePtrmap.iFirst. */ - typedef struct RollbackEntry RollbackEntry; typedef struct PtrmapEntry PtrmapEntry; struct PtrmapEntry { @@ -463,7 +458,6 @@ struct RollbackEntry { Pgno parent; u8 eType; }; - struct BtreePtrmap { Pgno iFirst; /* First new page number aPtr[0] */ @@ -479,17 +473,22 @@ struct BtreePtrmap { RollbackEntry *aRollback; /* Array of rollback entries */ }; +/* +** If page number pgno is greater than or equal to BtreePtrmap.iFirst, +** store an entry for it in the pointer-map structure. +*/ static int btreePtrmapStore( - BtreePtrmap *pMap, - Pgno pgno, + BtShared *pBt, + Pgno pgno, u8 eType, Pgno parent ){ + BtreePtrmap *pMap = pBt->pMap; if( pgno>=pMap->iFirst ){ int iEntry = pgno - pMap->iFirst; - /* Grow the aPtr[] array if required */ - if( iEntry>=pMap->nPtrAlloc ){ + /* Grow the aPtr[] array as required */ + while( iEntry>=pMap->nPtrAlloc ){ int nNew = pMap->nPtrAlloc ? pMap->nPtrAlloc*2 : 16; PtrmapEntry *aNew = (PtrmapEntry*)sqlite3_realloc( pMap->aPtr, nNew*sizeof(PtrmapEntry) @@ -535,8 +534,9 @@ static int btreePtrmapStore( /* ** Open savepoint iSavepoint, if it is not already open. */ -static int btreePtrmapBegin(BtreePtrmap *pMap, int nSvpt){ - if( nSvptnSvpt ){ +static int btreePtrmapBegin(BtShared *pBt, int nSvpt){ + BtreePtrmap *pMap = pBt->pMap; + if( pMap && nSvptnSvpt ){ int i; if( nSvpt>=pMap->nSvptAlloc ){ int nNew = pMap->nSvptAlloc ? pMap->nSvptAlloc*2 : 16; @@ -562,28 +562,72 @@ static int btreePtrmapBegin(BtreePtrmap *pMap, int nSvpt){ ** Rollback (if op==SAVEPOINT_ROLLBACK) or release (if op==SAVEPOINT_RELEASE) ** savepoint iSvpt. */ -static void btreePtrmapEnd(BtreePtrmap *pMap, int op, int iSvpt){ - assert( op==SAVEPOINT_ROLLBACK || op==SAVEPOINT_RELEASE ); - assert( iSvpt>=0 || (iSvpt==-1 && op==SAVEPOINT_ROLLBACK) ); - if( iSvpt<0 ){ - pMap->nSvpt = 0; - pMap->nRollback = 0; - memset(pMap->aPtr, 0, sizeof(Pgno) * pMap->nPtrAlloc); - }else if( iSvptnSvpt ){ - if( op==SAVEPOINT_ROLLBACK ){ - int ii; - for(ii=pMap->nRollback-1; ii>=pMap->aSvpt[iSvpt]; ii--){ - RollbackEntry *p = &pMap->aRollback[ii]; - PtrmapEntry *pEntry = &pMap->aPtr[p->pgno - pMap->iFirst]; - pEntry->parent = p->parent; - pEntry->eType = p->eType; +static void btreePtrmapEnd(BtShared *pBt, int op, int iSvpt){ + BtreePtrmap *pMap = pBt->pMap; + if( pMap ){ + assert( op==SAVEPOINT_ROLLBACK || op==SAVEPOINT_RELEASE ); + assert( iSvpt>=0 || (iSvpt==-1 && op==SAVEPOINT_ROLLBACK) ); + if( iSvpt<0 ){ + pMap->nSvpt = 0; + pMap->nRollback = 0; + memset(pMap->aPtr, 0, sizeof(Pgno) * pMap->nPtrAlloc); + }else if( iSvptnSvpt ){ + if( op==SAVEPOINT_ROLLBACK ){ + int ii; + for(ii=pMap->nRollback-1; ii>=pMap->aSvpt[iSvpt]; ii--){ + RollbackEntry *p = &pMap->aRollback[ii]; + PtrmapEntry *pEntry = &pMap->aPtr[p->pgno - pMap->iFirst]; + pEntry->parent = p->parent; + pEntry->eType = p->eType; + } } + pMap->nSvpt = iSvpt + (op==SAVEPOINT_ROLLBACK); + pMap->nRollback = pMap->aSvpt[iSvpt]; } - pMap->nSvpt = iSvpt + (op==SAVEPOINT_ROLLBACK); - pMap->nRollback = pMap->aSvpt[iSvpt]; } } +/* +** This function is called after an UNLOCKED transaction is opened on the +** database. It allocates the BtreePtrmap structure used to track pointers +** to allocated pages and zeroes the nFree/iTrunk fields in the database +** header on page 1. +*/ +static int btreePtrmapAllocate(BtShared *pBt){ + int rc = SQLITE_OK; + BtreePtrmap *pMap = sqlite3_malloc(sizeof(BtreePtrmap)); + assert( pBt->pMap==0 && sqlite3PagerIsUnlocked(pBt->pPager) ); + if( pMap==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(&pBt->pPage1->aData[32], 0, sizeof(u32)*2); + memset(pMap, 0, sizeof(BtreePtrmap)); + pMap->iFirst = pBt->nPage + 1; + pBt->pMap = pMap; + } + return rc; +} + +/* +** Free any BtreePtrmap structure allocated by an earlier call to +** btreePtrmapAllocate(). +*/ +static void btreePtrmapDelete(BtShared *pBt){ + BtreePtrmap *pMap = pBt->pMap; + if( pMap ){ + sqlite3_free(pMap->aRollback); + sqlite3_free(pMap->aPtr); + sqlite3_free(pMap->aSvpt); + sqlite3_free(pMap); + pBt->pMap = 0; + } +} +#else +# define btreePtrmapAllocate(x) SQLITE_OK +# define btreePtrmapDelete(x) +# define btreePtrmapBegin(x,y) SQLITE_OK +# define btreePtrmapEnd(x,y,z) +#endif static void releasePage(MemPage *pPage); /* Forward reference */ @@ -1024,44 +1068,49 @@ static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){ int offset; /* Offset in pointer map page */ int rc; /* Return code from subfunctions */ - assert( sqlite3_mutex_held(pBt->mutex) ); if( *pRC ) return; + assert( sqlite3_mutex_held(pBt->mutex) ); + /* The master-journal page number is never added to a pointer-map page */ + assert( 0==PTRMAP_ISPAGE(pBt, PENDING_BYTE_PAGE(pBt)) ); + +#ifdef SQLITE_ENABLE_UNLOCKED if( pBt->pMap ){ - *pRC = btreePtrmapStore(pBt->pMap, key, eType, parent); - }else{ - /* The master-journal page number must never be used as a ptr map page */ - assert( 0==PTRMAP_ISPAGE(pBt, PENDING_BYTE_PAGE(pBt)) ); + *pRC = btreePtrmapStore(pBt, key, eType, parent); + return; + } +#endif - assert( pBt->autoVacuum ); - if( key==0 ){ - *pRC = SQLITE_CORRUPT_BKPT; - return; - } - iPtrmap = PTRMAP_PAGENO(pBt, key); - rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage); - if( rc!=SQLITE_OK ){ - *pRC = rc; - return; - } - offset = PTRMAP_PTROFFSET(iPtrmap, key); - if( offset<0 ){ - *pRC = SQLITE_CORRUPT_BKPT; - }else{ - assert( offset <= (int)pBt->usableSize-5 ); - pPtrmap = (u8 *)sqlite3PagerGetData(pDbPage); + assert( pBt->autoVacuum ); + if( key==0 ){ + *pRC = SQLITE_CORRUPT_BKPT; + return; + } + iPtrmap = PTRMAP_PAGENO(pBt, key); + rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage); + if( rc!=SQLITE_OK ){ + *pRC = rc; + return; + } + offset = PTRMAP_PTROFFSET(iPtrmap, key); + if( offset<0 ){ + *pRC = SQLITE_CORRUPT_BKPT; + goto ptrmap_exit; + } + assert( offset <= (int)pBt->usableSize-5 ); + pPtrmap = (u8 *)sqlite3PagerGetData(pDbPage); - if( eType!=pPtrmap[offset] || get4byte(&pPtrmap[offset+1])!=parent ){ - TRACE(("PTRMAP_UPDATE: %d->(%d,%d)\n", key, eType, parent)); - *pRC= rc = sqlite3PagerWrite(pDbPage); - if( rc==SQLITE_OK ){ - pPtrmap[offset] = eType; - put4byte(&pPtrmap[offset+1], parent); - } - } + if( eType!=pPtrmap[offset] || get4byte(&pPtrmap[offset+1])!=parent ){ + TRACE(("PTRMAP_UPDATE: %d->(%d,%d)\n", key, eType, parent)); + *pRC= rc = sqlite3PagerWrite(pDbPage); + if( rc==SQLITE_OK ){ + pPtrmap[offset] = eType; + put4byte(&pPtrmap[offset+1], parent); } - sqlite3PagerUnref(pDbPage); } + +ptrmap_exit: + sqlite3PagerUnref(pDbPage); } /* @@ -2072,50 +2121,6 @@ u32 sqlite3BtreeLastPage(Btree *p){ return btreePagecount(p->pBt); } -#ifdef SQLITE_ENABLE_UNLOCKED -/* -** This function is called before allocating or freeing a b-tree page. If -** the current transaction is UNLOCKED, it allocates the BtreePtrmap -** structure and zeroes the nFree/iTrunk fields in the database header -** on page 1. -*/ -static int allocatePtrmap(BtShared *pBt){ - int rc = SQLITE_OK; - if( pBt->pMap==0 && sqlite3PagerIsUnlocked(pBt->pPager) ){ - /* If this is an unlocked transaction, set the header values - ** identifying the size of the free-list and the page number - ** of the first trunk page to zero. */ - BtreePtrmap *pMap = sqlite3_malloc(sizeof(BtreePtrmap)); - if( pMap==0 ){ - rc = SQLITE_NOMEM; - }else{ - memset(&pBt->pPage1->aData[32], 0, sizeof(u32)*2); - memset(pMap, 0, sizeof(BtreePtrmap)); - pMap->iFirst = btreePagecount(pBt) + 1; - pBt->pMap = pMap; - } - } - return rc; -} - -/* -** Free a pointer-map allocated by allocatePtrmap. -*/ -static void deletePtrmap(BtShared *pBt){ - BtreePtrmap *pMap = pBt->pMap; - if( pMap ){ - sqlite3_free(pMap->aRollback); - sqlite3_free(pMap->aPtr); - sqlite3_free(pMap->aSvpt); - sqlite3_free(pMap); - pBt->pMap = 0; - } -} -#else -#define allocatePtrmap(x) SQLITE_OK -#define deletePtrmap(x) -#endif - /* ** Get a page from the pager and initialize it. ** @@ -3341,8 +3346,8 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ if( rc==SQLITE_OK ){ rc = newDatabase(pBt); } - if( rc==SQLITE_OK ){ - rc = allocatePtrmap(pBt); + if( rc==SQLITE_OK && sqlite3PagerIsUnlocked(pBt->pPager) ){ + rc = btreePtrmapAllocate(pBt); } } } @@ -3402,8 +3407,8 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ */ int nSavepoint = p->db->nSavepoint; rc = sqlite3PagerOpenSavepoint(pBt->pPager, nSavepoint); - if( pBt->pMap && rc==SQLITE_OK && nSavepoint ){ - rc = btreePtrmapBegin(pBt->pMap, nSavepoint); + if( rc==SQLITE_OK && nSavepoint ){ + rc = btreePtrmapBegin(pBt, nSavepoint); } } @@ -3841,6 +3846,7 @@ static int autoVacuumCommit(BtShared *pBt){ # define setChildPtrmaps(x) SQLITE_OK #endif +#ifdef SQLITE_ENABLE_UNLOCKED /* ** The b-tree handle passed as the only argument is about to commit an ** UNLOCKED transaction. At this point it is guaranteed that this is @@ -3913,6 +3919,7 @@ static int btreeFixUnlocked(Btree *p){ Pgno iNew; /* New page number for pPg */ PtrmapEntry *pEntry; /* Pointer map entry for page iPg */ + if( iPg==PENDING_BYTE_PAGE(pBt) ) continue; pEntry = &pMap->aPtr[iPg - pMap->iFirst]; if( pEntry->eType==PTRMAP_FREEPAGE ){ MemPage *pFree = 0; @@ -3925,31 +3932,39 @@ static int btreeFixUnlocked(Btree *p){ assert( sqlite3PagerIswriteable(pPg->pDbPage) ); assert( sqlite3PagerPageRefcount(pPg->pDbPage)==1 ); iNew = ++nCurrent; + if( iNew==PENDING_BYTE_PAGE(pBt) ) iNew = ++nCurrent; rc = relocatePage(pBt, pPg, pEntry->eType, pEntry->parent, iNew, 1); releasePageNotNull(pPg); } } sqlite3PagerSetDbsize(pPager, nCurrent); + assert( nCurrent!=PENDING_BYTE_PAGE(pBt) ); nFree = get4byte(&p1[36]); nFinal = MAX(nCurrent-nFree, nHPage); + if( nCurrent>PENDING_BYTE_PAGE(pBt) && nFinal<=PENDING_BYTE_PAGE(pBt) ){ + nFinal--; + } - for(iPg=nFinal+1; rc==SQLITE_OK && iPgaPtr[iPg - pMap->iFirst]; if( pEntry->eType==PTRMAP_FREEPAGE ){ rc = allocateBtreePage(pBt, &pFree, &iNew, iPg, BTALLOC_EXACT); releasePage(pFree); }else{ rc = allocateBtreePage(pBt, &pFree, &iNew, nFinal, BTALLOC_LE); + assert( rc!=SQLITE_OK || iNew<=nFinal ); releasePage(pFree); if( rc==SQLITE_OK ){ MemPage *pPg = 0; btreeGetPage(pBt, iPg, &pPg, 0); rc = relocatePage(pBt, pPg, pEntry->eType, pEntry->parent,iNew,1); + releasePage(pPg); } } } @@ -3961,6 +3976,7 @@ static int btreeFixUnlocked(Btree *p){ return rc; } +#endif /* ** This routine does the first phase of a two-phase commit. This routine @@ -3995,11 +4011,8 @@ int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zMaster){ sqlite3BtreeEnter(p); #ifndef SQLITE_OMIT_AUTOVACUUM - /* Figure out if this is a commit of an UNLOCKED transaction that - ** requires a snapshot upgrade. If so, skip any auto-vacuum - ** processing. */ if( pBt->autoVacuum ){ - assert( sqlite3PagerIsUnlocked(pBt->pPager)==0 ); + assert( ISUNLOCKED==0 ); rc = autoVacuumCommit(pBt); if( rc!=SQLITE_OK ){ sqlite3BtreeLeave(p); @@ -4010,7 +4023,7 @@ int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zMaster){ sqlite3PagerTruncateImage(pBt->pPager, pBt->nPage); } #endif - if( rc==SQLITE_OK && sqlite3PagerIsUnlocked(pBt->pPager) ){ + if( rc==SQLITE_OK && ISUNLOCKED ){ rc = btreeFixUnlocked(p); } if( rc==SQLITE_OK ){ @@ -4059,7 +4072,7 @@ static void btreeEndTransaction(Btree *p){ } /* If this was an UNLOCKED transaction, delete the pBt->pMap object */ - deletePtrmap(pBt); + btreePtrmapDelete(pBt); btreeIntegrity(p); } @@ -4285,8 +4298,8 @@ int sqlite3BtreeBeginStmt(Btree *p, int iStatement){ ** such savepoints while the statement transaction savepoint is active. */ rc = sqlite3PagerOpenSavepoint(pBt->pPager, iStatement); - if( rc==SQLITE_OK && pBt->pMap ){ - rc = btreePtrmapBegin(pBt->pMap, iStatement); + if( rc==SQLITE_OK ){ + rc = btreePtrmapBegin(pBt, iStatement); } sqlite3BtreeLeave(p); return rc; @@ -4311,7 +4324,7 @@ int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK ); assert( iSavepoint>=0 || (iSavepoint==-1 && op==SAVEPOINT_ROLLBACK) ); sqlite3BtreeEnter(p); - if( pBt->pMap ) btreePtrmapEnd(pBt->pMap, op, iSavepoint); + btreePtrmapEnd(pBt, op, iSavepoint); rc = sqlite3PagerSavepoint(pBt->pPager, op, iSavepoint); if( rc==SQLITE_OK ){ if( iSavepoint<0 && (pBt->btsFlags & BTS_INITIALLY_EMPTY)!=0 ){ @@ -5817,7 +5830,7 @@ static int allocateBtreePage( ** stores stores the total number of pages on the freelist. */ n = get4byte(&pPage1->aData[36]); testcase( n==mxPage-1 ); - if( sqlite3PagerIsUnlocked(pBt->pPager)==0 && n>=mxPage ){ + if( ISUNLOCKED==0 && n>=mxPage ){ return SQLITE_CORRUPT_BKPT; } @@ -5840,7 +5853,7 @@ static int allocateBtreePage( ** the entire-list will be searched for that page. */ if( eMode==BTALLOC_EXACT ){ - assert( ISAUTOVACUUM==!sqlite3PagerIsUnlocked(pBt->pPager) ); + assert( ISAUTOVACUUM!=ISUNLOCKED ); if( ISAUTOVACUUM ){ if( nearby<=mxPage ){ u8 eType; @@ -9926,6 +9939,23 @@ int sqlite3BtreeIsReadonly(Btree *p){ */ int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); } +/* +** This function is called to ensure that all locks required to commit the +** current write-transaction to the database file are held. If the db is +** in rollback mode, this means the EXCLUSIVE lock on the database file. +** +** Or, if this is an UNLOCKED transaction on a wal-mode database, the WRITER +** lock on the wal file. In this case this function also checks that the +** UNLOCKED transaction can be safely committed (does not commit with any +** other transaction committed since it was opened). +** +** SQLITE_OK is returned if successful. SQLITE_BUSY if the required locks +** cannot be obtained due to a conflicting lock. If the locks cannot be +** obtained for an UNLOCKED transaction due to a conflict with an already +** committed transaction, SQLITE_BUSY_SNAPSHOT is returned. Otherwise, if +** some other error (OOM, IO, etc.) occurs, the relevant SQLite error code +** is returned. +*/ int sqlite3BtreeExclusiveLock(Btree *p){ int rc; BtShared *pBt = p->pBt; diff --git a/src/btreeInt.h b/src/btreeInt.h index a60f2d5c17..4c23629cc8 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -664,11 +664,12 @@ struct BtCursor { #endif #ifdef SQLITE_ENABLE_UNLOCKED -# define REQUIRE_PTRMAP (ISAUTOVACUUM || pBt->pMap) +# define ISUNLOCKED (pBt->pMap!=0) #else -# define REQUIRE_PTRMAP ISAUTOVACUUM +# define ISUNLOCKED 0 #endif +#define REQUIRE_PTRMAP (ISAUTOVACUUM || ISUNLOCKED) /* ** This structure is passed around through all the sanity checking routines diff --git a/test/unlocked2.test b/test/unlocked2.test index e5b102aa02..927fe3e675 100644 --- a/test/unlocked2.test +++ b/test/unlocked2.test @@ -68,7 +68,6 @@ do_multiclient_test tn { sql2 { INSERT INTO t2 VALUES(8); } - breakpoint sql1 { COMMIT; } @@ -225,5 +224,38 @@ do_multiclient_test tn { } {} } +#------------------------------------------------------------------------- +# +do_multiclient_test tn { + do_test 5.$tn.1 { + sql1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x); + CREATE TABLE t2(x); + INSERT INTO t1 VALUES(randomblob(1500)); + PRAGMA page_count; + } + } {wal 4} + + do_test 5.$tn.2 { + sql1 { + BEGIN UNLOCKED; + INSERT INTO t2 VALUES(randomblob(1500)); + PRAGMA page_count; + } + } {5} + + do_test 5.$tn.3 { + sql2 { + DELETE FROM t1; + PRAGMA freelist_count; + PRAGMA page_count; + } + } {1 4} + + do_test 5.$tn.4 { sql1 COMMIT } {} + do_test 5.$tn.5 { sql3 { PRAGMA integrity_check } } {ok} +} + finish_test diff --git a/test/unlocked3.test b/test/unlocked3.test new file mode 100644 index 0000000000..04e741dc5b --- /dev/null +++ b/test/unlocked3.test @@ -0,0 +1,110 @@ +# 2015 July 26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests for transactions started with BEGIN UNLOCKED. The tests in this +# file focus on testing that deferred page allocation works properly. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set ::testprefix unlocked3 + +if {$AUTOVACUUM} { finish_test ; return } + +proc create_schema {} { + db eval { + PRAGMA journal_mode = wal; + + CREATE TABLE t1(x, y); + CREATE TABLE t2(x, y); + CREATE TABLE t3(x, y); + CREATE TABLE t4(x, y); + + CREATE INDEX i1 ON t1(y, x); + CREATE INDEX i2 ON t2(y, x); + CREATE INDEX i3 ON t3(y, x); + CREATE INDEX i4 ON t4(y, x); + } +} + +proc do_sql_op {iTbl iOp} { + set db "db$iTbl" + + switch $iOp { + "i" { + set sql " + WITH cnt(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM cnt WHERE i<10) + INSERT INTO t$iTbl SELECT randomblob(800), randomblob(800) FROM cnt; + " + } + + "d" { + set sql " + DELETE FROM t$iTbl WHERE rowid IN ( + SELECT rowid FROM t$iTbl ORDER BY 1 ASC LIMIT 10 + ) + " + } + + default { + error "bad iOp parameter: $iOp" + } + } + + $db eval $sql +} + + +set DBLIST {db1 db2 db3 db4} + +create_schema +foreach {tn oplist} { + 1 {1i 2i 3i 4i} + 2 {1iii 2iii 3iii 4iii} + 3 {1d 2d 3d 4d} + . ----------------------- + 4 {1i} + 5 {1d 2i} + . ----------------------- + 6 {1iii 2iii 3iii 4iii} + 7 {1di 2id 3iii 4ddd} + 8 {1iii 2iii 3iii 4iii} +} { + if {[string range $oplist 0 0]=="-"} { + reset_db + create_schema + continue + } + foreach db $DBLIST { sqlite3 $db test.db } + + do_test 1.$tn { + foreach db $DBLIST { $db eval "BEGIN UNLOCKED" } + + foreach op $oplist { + set iTbl [string range $op 0 0] + foreach char [split [string range $op 1 end] {}] { + do_sql_op $iTbl $char + } + } + + foreach db $DBLIST { $db eval "COMMIT" } + db eval {PRAGMA integrity_check} + } {ok} + + foreach db $DBLIST { + if {$db=="db2"} breakpoint + $db close + } +} + +finish_test + From 0c52b373a0d3606ca4475f506d67be9c568a7186 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 22 Aug 2015 20:32:39 +0000 Subject: [PATCH 012/179] Fix a problem to do with detecting unlocked transaction conflicts if another client restarts the wal while the transaction is running. FossilOrigin-Name: e3968b256282d8c0a87c26667b1ba02faf7a5a17 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/wal.c | 13 +++++++------ test/unlocked2.test | 35 +++++++++++++++++++++++++++++++++++ test/unlocked3.test | 18 ++++++++++++++++++ 5 files changed, 69 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index bc04296ba0..1778b94253 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sfurther\stests\sfor\sdeferred\spage\sallocation.\sAnd\sfixes\sfor\sthe\ssame. -D 2015-08-22T17:28:55.533 +C Fix\sa\sproblem\sto\sdo\swith\sdetecting\sunlocked\stransaction\sconflicts\sif\sanother\sclient\srestarts\sthe\swal\swhile\sthe\stransaction\sis\srunning. +D 2015-08-22T20:32:39.272 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -411,7 +411,7 @@ F src/vdbesort.c f5009e7a35e3065635d8918b9a31f498a499976b F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 F src/vtab.c d31174e4c8f592febab3fa7f69e18320b4fd657a F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb -F src/wal.c 0ba4899f02457be91fe5bc734638fee00c807f64 +F src/wal.c 4c69d27ed7ec7006008d8309dc5c9ab742273c12 F src/wal.h a701096b9bf5761b8c5f4b07082249e1950afc9a F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba F src/where.c 66518a14a1238611aa0744d6980b6b7f544f4816 @@ -1228,8 +1228,8 @@ F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unique2.test 41e7f83c6827605991160a31380148a9fc5f1339 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 F test/unlocked.test d143a871bd874311d4e5c98ce458218ca6b579fd -F test/unlocked2.test 7f913b404cda03f22bc7b299e8eb82a34d2880c5 -F test/unlocked3.test 4facd06286885faa9bd53713fa3d701ca403aeb2 +F test/unlocked2.test aaa42a08052a146466220b6c3a062ff3c4776ad6 +F test/unlocked3.test 06173e406a8c1a550448f79329ed4c3311905062 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c2327a3b8e5d604ab948b1e9f6cfc401429e51db -R c62a3f455b34280d099a1332c1c4f1e4 +P ed0a31be726e60115a5dd73d4ed580201b400ab7 +R 4d5e4d57de949c83b8660752ea3918ee U dan -Z 1442e612d3eba52580ab50755ceb943f +Z 75094882a86841f2760b97ee8543ae22 diff --git a/manifest.uuid b/manifest.uuid index 545070a432..c18cc52736 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ed0a31be726e60115a5dd73d4ed580201b400ab7 \ No newline at end of file +e3968b256282d8c0a87c26667b1ba02faf7a5a17 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index a9d8a44086..fcf5a15044 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2594,13 +2594,14 @@ int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPage1, Bitvec *pAllRead){ /* TODO: Check header checksum is good here. */ if( memcmp(&pWal->hdr, (void*)pHead, sizeof(WalIndexHdr))!=0 ){ - /* TODO: Is this safe? Because it holds the WRITER lock this thread - ** has exclusive access to the live header, but might it be corrupt? - ** This code should check that the wal-index-header is Ok, and return - ** SQLITE_BUSY_SNAPSHOT if it is not. */ int iHash; int iLastHash = walFramePage(pHead->mxFrame); - for(iHash=walFramePage(pWal->hdr.mxFrame+1); iHash<=iLastHash; iHash++){ + u32 iFirst = pWal->hdr.mxFrame+1; /* First wal frame to check */ + if( memcmp(pWal->hdr.aSalt, (u32*)pHead->aSalt, sizeof(u32)*2) ){ + assert( pWal->readLock==0 ); + iFirst = 1; + } + for(iHash=walFramePage(iFirst); iHash<=iLastHash; iHash++){ volatile ht_slot *aHash; volatile u32 *aPgno; u32 iZero; @@ -2608,7 +2609,7 @@ int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPage1, Bitvec *pAllRead){ rc = walHashGet(pWal, iHash, &aHash, &aPgno, &iZero); if( rc==SQLITE_OK ){ int i; - int iMin = (pWal->hdr.mxFrame+1 - iZero); + int iMin = (iFirst - iZero); int iMax = (iHash==0) ? HASHTABLE_NPAGE_ONE : HASHTABLE_NPAGE; if( iMin<1 ) iMin = 1; if( iMax>pHead->mxFrame ) iMax = pHead->mxFrame; diff --git a/test/unlocked2.test b/test/unlocked2.test index 927fe3e675..6953753643 100644 --- a/test/unlocked2.test +++ b/test/unlocked2.test @@ -257,5 +257,40 @@ do_multiclient_test tn { do_test 5.$tn.5 { sql3 { PRAGMA integrity_check } } {ok} } +#------------------------------------------------------------------------- +# +do_multiclient_test tn { + do_test 6.$tn.1 { + sql1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(randomblob(1500)); + PRAGMA wal_checkpoint; + } + } {wal 0 5 5} + + do_test 6.$tn.2 { + sql1 { + BEGIN UNLOCKED; + INSERT INTO t1 VALUES(randomblob(1500)); + INSERT INTO t1 VALUES(randomblob(1500)); + } + } {} + + do_test 6.$tn.3 { + sql2 { + BEGIN; + INSERT INTO t1 VALUES(randomblob(1500)); + INSERT INTO t1 VALUES(randomblob(1500)); + COMMIT; + } + } {} + + do_test 6.$tn.4 { + list [catch { sql1 COMMIT } msg] $msg + } {1 {database is locked}} + do_test 6.$tn.5 { sql3 { PRAGMA integrity_check } } {ok} + do_test 6.$tn.5 { sql3 { SELECT count(*) from t1 } } {3} +} finish_test diff --git a/test/unlocked3.test b/test/unlocked3.test index 04e741dc5b..b4deaf9a92 100644 --- a/test/unlocked3.test +++ b/test/unlocked3.test @@ -55,6 +55,22 @@ proc do_sql_op {iTbl iOp} { " } + "D" { + set sql " + DELETE FROM t$iTbl WHERE rowid IN ( + SELECT rowid FROM t$iTbl o WHERE ( + SELECT count(*) FROM t$iTbl i WHERE i.rowid Date: Mon, 24 Aug 2015 06:43:25 +0000 Subject: [PATCH 013/179] Fix another problem involving unlocked transactions and wal-file restarts. FossilOrigin-Name: 4460764ea8fc948fe02f0a09476857839b3aa1ae --- manifest | 18 +++++++------- manifest.uuid | 2 +- src/pager.c | 6 +++-- src/wal.c | 59 +++++++++++++++++++++++++++++++++++---------- src/wal.h | 2 +- test/unlocked3.test | 4 +-- 6 files changed, 62 insertions(+), 29 deletions(-) diff --git a/manifest b/manifest index 1778b94253..1285f64e76 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sto\sdo\swith\sdetecting\sunlocked\stransaction\sconflicts\sif\sanother\sclient\srestarts\sthe\swal\swhile\sthe\stransaction\sis\srunning. -D 2015-08-22T20:32:39.272 +C Fix\sanother\sproblem\sinvolving\sunlocked\stransactions\sand\swal-file\srestarts. +D 2015-08-24T06:43:25.273 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -324,7 +324,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 388c023582b17890f10c980b30ec1922b471753b F src/os_win.c 40b3af7a47eb1107d0d69e592bec345a3b7b798a F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 951b7bae30fcbc2a2d380fa68f8bc68286ae0760 +F src/pager.c a952e4b4c2e85e1588df5972688de9b6c989dbdd F src/pager.h 1e7b0fc3846b71bd95b4b3300820d756895cb4ef F src/parse.y e9accdb2cb1795f75f478e7ce89e17b19d7d4da7 F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0 @@ -411,8 +411,8 @@ F src/vdbesort.c f5009e7a35e3065635d8918b9a31f498a499976b F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 F src/vtab.c d31174e4c8f592febab3fa7f69e18320b4fd657a F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb -F src/wal.c 4c69d27ed7ec7006008d8309dc5c9ab742273c12 -F src/wal.h a701096b9bf5761b8c5f4b07082249e1950afc9a +F src/wal.c b8811f666d785820437d95386c12b85951418c50 +F src/wal.h 903ef67e17f8b466dc7cfc4186fc23e80be10ff8 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba F src/where.c 66518a14a1238611aa0744d6980b6b7f544f4816 F src/whereInt.h 880a8599226ac1c00203490d934f3ed79b292572 @@ -1229,7 +1229,7 @@ F test/unique2.test 41e7f83c6827605991160a31380148a9fc5f1339 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 F test/unlocked.test d143a871bd874311d4e5c98ce458218ca6b579fd F test/unlocked2.test aaa42a08052a146466220b6c3a062ff3c4776ad6 -F test/unlocked3.test 06173e406a8c1a550448f79329ed4c3311905062 +F test/unlocked3.test 1bc4e3ad8693302639e50b4833a527807ee4863b F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ed0a31be726e60115a5dd73d4ed580201b400ab7 -R 4d5e4d57de949c83b8660752ea3918ee +P e3968b256282d8c0a87c26667b1ba02faf7a5a17 +R 616e25ded22fb9d47c944cb40768f03a U dan -Z 75094882a86841f2760b97ee8543ae22 +Z 4efd00eba0478d425d47255076a7c364 diff --git a/manifest.uuid b/manifest.uuid index c18cc52736..75583c940a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e3968b256282d8c0a87c26667b1ba02faf7a5a17 \ No newline at end of file +4460764ea8fc948fe02f0a09476857839b3aa1ae \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index c55db40150..c2344e3747 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6129,8 +6129,10 @@ int sqlite3PagerUpgradeSnapshot(Pager *pPager, DbPage *pPage1){ u32 iFrame = 0; assert( pPager->pWal && pPager->pAllRead ); - sqlite3WalUpgradeSnapshot(pPager->pWal); - rc = sqlite3WalFindFrame(pPager->pWal, 1, &iFrame); + rc = sqlite3WalUpgradeSnapshot(pPager->pWal); + if( rc==SQLITE_OK ){ + rc = sqlite3WalFindFrame(pPager->pWal, 1, &iFrame); + } if( rc==SQLITE_OK ){ rc = readDbPage(pPage1, iFrame); } diff --git a/src/wal.c b/src/wal.c index fcf5a15044..d43f939f00 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2667,9 +2667,44 @@ int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPage1, Bitvec *pAllRead){ return rc; } -void sqlite3WalUpgradeSnapshot(Wal *pWal){ +/* +** This function is called by a writer that has a read-lock on aReadmark[0] +** (pWal->readLock==0). This function relinquishes that lock and takes a +** lock on a different aReadmark[] slot. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +static int walUpgradeReadlock(Wal *pWal){ + int cnt; + int rc; + assert( pWal->writeLock && pWal->readLock==0 ); + walUnlockShared(pWal, WAL_READ_LOCK(0)); + pWal->readLock = -1; + cnt = 0; + do{ + int notUsed; + rc = walTryBeginRead(pWal, ¬Used, 1, ++cnt); + }while( rc==WAL_RETRY ); + assert( (rc&0xff)!=SQLITE_BUSY ); /* BUSY not possible when useWal==1 */ + testcase( (rc&0xff)==SQLITE_IOERR ); + testcase( rc==SQLITE_PROTOCOL ); + testcase( rc==SQLITE_OK ); + return rc; +} + +int sqlite3WalUpgradeSnapshot(Wal *pWal){ + int rc = SQLITE_OK; assert( pWal->writeLock ); memcpy(&pWal->hdr, (void*)walIndexHdr(pWal), sizeof(WalIndexHdr)); + + /* If this client has its read-lock on slot aReadmark[0] and the entire + ** wal has not been checkpointed, switch it to a different slot. Otherwise + ** any reads performed between now and committing the transaction will + ** read from the old snapshot - not the one just upgraded to. */ + if( pWal->readLock==0 && pWal->hdr.mxFrame!=walCkptInfo(pWal)->nBackfill ){ + rc = walUpgradeReadlock(pWal); + } + return rc; } /* @@ -2789,7 +2824,6 @@ int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ */ static int walRestartLog(Wal *pWal){ int rc = SQLITE_OK; - int cnt; if( pWal->readLock==0 ){ volatile WalCkptInfo *pInfo = walCkptInfo(pWal); @@ -2814,17 +2848,16 @@ static int walRestartLog(Wal *pWal){ return rc; } } - walUnlockShared(pWal, WAL_READ_LOCK(0)); - pWal->readLock = -1; - cnt = 0; - do{ - int notUsed; - rc = walTryBeginRead(pWal, ¬Used, 1, ++cnt); - }while( rc==WAL_RETRY ); - assert( (rc&0xff)!=SQLITE_BUSY ); /* BUSY not possible when useWal==1 */ - testcase( (rc&0xff)==SQLITE_IOERR ); - testcase( rc==SQLITE_PROTOCOL ); - testcase( rc==SQLITE_OK ); + + /* Regardless of whether or not the wal file was restarted, change the + ** read-lock held by this client to a slot other than aReadmark[0]. + ** Clients with a lock on aReadmark[0] read from the database file + ** only - never from the wal file. This means that if a writer holding + ** a lock on aReadmark[0] were to commit a transaction but not close the + ** read-transaction, subsequent read operations would read directly from + ** the database file - ignoring the new pages just appended + ** to the wal file. */ + rc = walUpgradeReadlock(pWal); } return rc; } diff --git a/src/wal.h b/src/wal.h index 0609eded27..35ffe4d38c 100644 --- a/src/wal.h +++ b/src/wal.h @@ -127,7 +127,7 @@ int sqlite3WalExclusiveMode(Wal *pWal, int op); int sqlite3WalHeapMemory(Wal *pWal); int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPg, Bitvec *pRead); -void sqlite3WalUpgradeSnapshot(Wal *pWal); +int sqlite3WalUpgradeSnapshot(Wal *pWal); #ifdef SQLITE_ENABLE_ZIPVFS /* If the WAL file is not empty, return the number of bytes of content diff --git a/test/unlocked3.test b/test/unlocked3.test index b4deaf9a92..9cde078833 100644 --- a/test/unlocked3.test +++ b/test/unlocked3.test @@ -94,9 +94,8 @@ foreach {tn oplist} { 6 {1iii 2iii 3iii 4iii} 7 {1di 2id 3iii 4ddd} 8 {1iii 2iii 3iii 4iii} - + 9 {1D 2II} } { - # 9 {1D 2II} if {[string range $oplist 0 0]=="-"} { reset_db create_schema @@ -119,7 +118,6 @@ foreach {tn oplist} { } {ok} foreach db $DBLIST { - if {$db=="db2"} breakpoint $db close } } From 5cf03728a6d33823bb55cbab69f5e4b415e4d1ae Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 24 Aug 2015 10:05:03 +0000 Subject: [PATCH 014/179] Consolidate two blocks of similar code in btreeFixUnlocked(). FossilOrigin-Name: 701302b4bd62ca7aefe643eac096a0ee672a62fa --- manifest | 14 ++--- manifest.uuid | 2 +- src/btree.c | 142 ++++++++++++++++++++++++++------------------ test/unlocked3.test | 24 ++++---- 4 files changed, 106 insertions(+), 76 deletions(-) diff --git a/manifest b/manifest index 06617c4d57..fbf1b20715 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\swith\sthis\sbranch. -D 2015-08-24T06:44:17.840 +C Consolidate\stwo\sblocks\sof\ssimilar\scode\sin\sbtreeFixUnlocked(). +D 2015-08-24T10:05:03.012 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -279,7 +279,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 4d9134dc988a87838c06056c89c0e8c4700a0452 F src/bitvec.c d1f21d7d91690747881f03940584f4cc548c9d3d F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 -F src/btree.c 245bc696824d5107f3570905325c6ac5b05c18f8 +F src/btree.c e807409602996359211c855078fa5cbcf59da032 F src/btree.h 00d4cdb747c4172a5566faf037116985dbbc377e F src/btreeInt.h 0c19847f87ab82e4f5e67750a069f10829475da6 F src/build.c e47b6fffe14a28d9050e6747beebb01597d37542 @@ -1229,7 +1229,7 @@ F test/unique2.test 41e7f83c6827605991160a31380148a9fc5f1339 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 F test/unlocked.test d143a871bd874311d4e5c98ce458218ca6b579fd F test/unlocked2.test aaa42a08052a146466220b6c3a062ff3c4776ad6 -F test/unlocked3.test 1bc4e3ad8693302639e50b4833a527807ee4863b +F test/unlocked3.test 2cacc31742561f6924706ed9472d0095839da9e4 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4460764ea8fc948fe02f0a09476857839b3aa1ae f0aba0e120074430cd7ad93291fcc97b8a25a054 -R c9a4c2c74a91d952dffb223e6caf31f7 +P 876810c28b3ad573ae46050ec699ef7eea4e313d +R 11d77b8f4d3f6d9b5ad54020bb4092ee U dan -Z 9a4849fb712895f27dbf139db01b2763 +Z 46d9657b2162cb9e544c380eb188ec18 diff --git a/manifest.uuid b/manifest.uuid index 9e718e4bc3..93c1197630 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -876810c28b3ad573ae46050ec699ef7eea4e313d \ No newline at end of file +701302b4bd62ca7aefe643eac096a0ee672a62fa \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 53b769720d..0b6b1305ae 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3847,6 +3847,73 @@ static int autoVacuumCommit(BtShared *pBt){ #endif #ifdef SQLITE_ENABLE_UNLOCKED +/* +** This function is called as part of merging an UNLOCKED transaction with +** the snapshot at the head of the wal file. It relocates all pages in the +** range iFirst..iLast, inclusive. It is assumed that the BtreePtrmap +** structure at BtShared.pMap contains the location of the pointers to each +** page in the range. +** +** If pnCurrent is NULL, then all pages in the range are moved to currently +** free locations (i.e. free-list entries) within the database file before page +** iFirst. +** +** Or, if pnCurrent is not NULL, then it points to a value containing the +** current size of the database file in pages. In this case, all pages are +** relocated to the end of the database file - page iFirst is relocated to +** page (*pnCurrent+1), page iFirst+1 to page (*pnCurrent+2), and so on. +** Value *pnCurrent is set to the new size of the database before this +** function returns. +** +** If no error occurs, SQLITE_OK is returned. Otherwise, an SQLite error code. +*/ +static int btreeRelocateRange( + BtShared *pBt, /* B-tree handle */ + Pgno iFirst, /* First page to relocate */ + Pgno iLast, /* Last page to relocate */ + Pgno *pnCurrent /* If not NULL, IN/OUT: Database size */ +){ + int rc = SQLITE_OK; + BtreePtrmap *pMap = pBt->pMap; + Pgno iPg; + + for(iPg=iFirst; iPg<=iLast && rc==SQLITE_OK; iPg++){ + MemPage *pFree = 0; /* Page allocated from free-list */ + MemPage *pPg = 0; + Pgno iNew; /* New page number for pPg */ + PtrmapEntry *pEntry; /* Pointer map entry for page iPg */ + + if( iPg==PENDING_BYTE_PAGE(pBt) ) continue; + pEntry = &pMap->aPtr[iPg - pMap->iFirst]; + + if( pEntry->eType==PTRMAP_FREEPAGE ){ + Pgno dummy; + rc = allocateBtreePage(pBt, &pFree, &dummy, iPg, BTALLOC_EXACT); + releasePage(pFree); + assert( rc!=SQLITE_OK || dummy==iPg ); + }else if( pnCurrent ){ + btreeGetPage(pBt, iPg, &pPg, 0); + assert( sqlite3PagerIswriteable(pPg->pDbPage) ); + assert( sqlite3PagerPageRefcount(pPg->pDbPage)==1 ); + iNew = ++(*pnCurrent); + if( iNew==PENDING_BYTE_PAGE(pBt) ) iNew = ++(*pnCurrent); + rc = relocatePage(pBt, pPg, pEntry->eType, pEntry->parent, iNew, 1); + releasePageNotNull(pPg); + }else{ + rc = allocateBtreePage(pBt, &pFree, &iNew, iFirst-1, BTALLOC_LE); + assert( rc!=SQLITE_OK || iNeweType, pEntry->parent,iNew,1); + releasePage(pPg); + } + } + } + return rc; +} + /* ** The b-tree handle passed as the only argument is about to commit an ** UNLOCKED transaction. At this point it is guaranteed that this is @@ -3875,7 +3942,7 @@ static int btreeFixUnlocked(Btree *p){ if( rc==SQLITE_OK ){ Pgno nHPage = get4byte(&p1[28]); - Pgno nFinal = nHPage; /* Size of db after transaction merge */ + Pgno nFin = nHPage; /* Size of db after transaction merge */ if( sqlite3PagerIswriteable(pPage1->pDbPage) ){ Pgno iHTrunk = get4byte(&p1[32]); @@ -3907,71 +3974,32 @@ static int btreeFixUnlocked(Btree *p){ /* The current transaction allocated pages pMap->iFirst through ** nPage (inclusive) at the end of the database file. Meanwhile, ** other transactions have allocated (iFirst..nHPage). So move - ** pages (iFirst..MIN(nPage,nHPage)) to (MAX(nPage,nHPage)+1). - */ + ** pages (iFirst..MIN(nPage,nHPage)) to (MAX(nPage,nHPage)+1). */ Pgno iLast = MIN(nPage, nHPage); /* Last page to move */ - Pgno iPg; Pgno nCurrent; /* Current size of db */ nCurrent = MAX(nPage, nHPage); + rc = btreeRelocateRange(pBt, pMap->iFirst, iLast, &nCurrent); - for(iPg=pMap->iFirst; iPg<=iLast && rc==SQLITE_OK; iPg++){ - MemPage *pPg = 0; - Pgno iNew; /* New page number for pPg */ - PtrmapEntry *pEntry; /* Pointer map entry for page iPg */ - - if( iPg==PENDING_BYTE_PAGE(pBt) ) continue; - pEntry = &pMap->aPtr[iPg - pMap->iFirst]; - if( pEntry->eType==PTRMAP_FREEPAGE ){ - MemPage *pFree = 0; - Pgno dummy; - rc = allocateBtreePage(pBt, &pFree, &dummy, iPg, BTALLOC_EXACT); - releasePage(pFree); - assert( rc!=SQLITE_OK || dummy==iPg ); - }else{ - btreeGetPage(pBt, iPg, &pPg, 0); - assert( sqlite3PagerIswriteable(pPg->pDbPage) ); - assert( sqlite3PagerPageRefcount(pPg->pDbPage)==1 ); - iNew = ++nCurrent; - if( iNew==PENDING_BYTE_PAGE(pBt) ) iNew = ++nCurrent; - rc = relocatePage(pBt, pPg, pEntry->eType, pEntry->parent, iNew, 1); - releasePageNotNull(pPg); + /* There are now no collisions with the snapshot at the head of the + ** database file. So at this point it would be possible to write + ** the transaction out to disk. Before doing so though, attempt to + ** relocate some of the new pages to free locations within the body + ** of the database file (i.e. free-list entries). */ + if( rc==SQLITE_OK ){ + assert( nCurrent!=PENDING_BYTE_PAGE(pBt) ); + sqlite3PagerSetDbsize(pBt->pPager, nCurrent); + nFree = get4byte(&p1[36]); + nFin = MAX(nCurrent-nFree, nHPage); + if( nCurrent>PENDING_BYTE_PAGE(pBt) && nFin<=PENDING_BYTE_PAGE(pBt) ){ + nFin--; } - } - sqlite3PagerSetDbsize(pPager, nCurrent); - assert( nCurrent!=PENDING_BYTE_PAGE(pBt) ); - - nFree = get4byte(&p1[36]); - nFinal = MAX(nCurrent-nFree, nHPage); - if( nCurrent>PENDING_BYTE_PAGE(pBt) && nFinal<=PENDING_BYTE_PAGE(pBt) ){ - nFinal--; + rc = btreeRelocateRange(pBt, nFin+1, nCurrent, 0); } - for(iPg=nFinal+1; rc==SQLITE_OK && iPg<=nCurrent; iPg++){ - Pgno iNew; /* New page number for pPg */ - MemPage *pFree; - PtrmapEntry *pEntry; /* Pointer map entry for page iPg */ - - if( iPg==PENDING_BYTE_PAGE(pBt) ) continue; - pEntry = &pMap->aPtr[iPg - pMap->iFirst]; - if( pEntry->eType==PTRMAP_FREEPAGE ){ - rc = allocateBtreePage(pBt, &pFree, &iNew, iPg, BTALLOC_EXACT); - releasePage(pFree); - }else{ - rc = allocateBtreePage(pBt, &pFree, &iNew, nFinal, BTALLOC_LE); - assert( rc!=SQLITE_OK || iNew<=nFinal ); - releasePage(pFree); - if( rc==SQLITE_OK ){ - MemPage *pPg = 0; - btreeGetPage(pBt, iPg, &pPg, 0); - rc = relocatePage(pBt, pPg, pEntry->eType, pEntry->parent,iNew,1); - releasePage(pPg); - } - } - } - put4byte(&p1[28], nFinal); + put4byte(&p1[28], nFin); } } - sqlite3PagerSetDbsize(pPager, nFinal); + sqlite3PagerSetDbsize(pPager, nFin); } return rc; diff --git a/test/unlocked3.test b/test/unlocked3.test index 9cde078833..61e71c87c3 100644 --- a/test/unlocked3.test +++ b/test/unlocked3.test @@ -84,17 +84,19 @@ set DBLIST {db1 db2 db3 db4} create_schema foreach {tn oplist} { - 1 {1i 2i 3i 4i} - 2 {1iii 2iii 3iii 4iii} - 3 {1d 2d 3d 4d} - . ----------------------- - 4 {1i} - 5 {1d 2i} - . ----------------------- - 6 {1iii 2iii 3iii 4iii} - 7 {1di 2id 3iii 4ddd} - 8 {1iii 2iii 3iii 4iii} - 9 {1D 2II} + 1 {1i 2i 3i 4i} + 2 {1iii 2iii 3iii 4iii} + 3 {1d 2d 3d 4d} + . ----------------------- + 4 {1i} + 5 {1d 2i} + . ----------------------- + 6 {1iii 2iii 3iii 4iii} + 7 {1di 2id 3iii 4ddd} + 8 {1iii 2iii 3iii 4iii} + 9 {1D 2II} + 10 {1I 2D 3I 4D} + 11 {1III 3dddddd 4III} } { if {[string range $oplist 0 0]=="-"} { reset_db From 0408529b48af957c0860fc61848e1b24386c7332 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 24 Aug 2015 16:00:08 +0000 Subject: [PATCH 015/179] Fix compilation without SQLITE_ENABLE_UNLOCKED. Also other code organization issues. FossilOrigin-Name: 041135575417201bbcf0544cc69dcb7369c7fb34 --- manifest | 26 +++++------ manifest.uuid | 2 +- src/btree.c | 2 + src/pager.c | 107 ++++++++++++++++++++++++++++++-------------- src/test_config.c | 6 +++ src/vdbeaux.c | 2 + src/wal.c | 84 ++++++++++++++++++++++++---------- test/unlocked.test | 4 ++ test/unlocked2.test | 4 ++ test/unlocked3.test | 4 ++ 10 files changed, 170 insertions(+), 71 deletions(-) diff --git a/manifest b/manifest index fbf1b20715..7e91525501 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Consolidate\stwo\sblocks\sof\ssimilar\scode\sin\sbtreeFixUnlocked(). -D 2015-08-24T10:05:03.012 +C Fix\scompilation\swithout\sSQLITE_ENABLE_UNLOCKED.\sAlso\sother\scode\sorganization\sissues. +D 2015-08-24T16:00:08.023 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -279,7 +279,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 4d9134dc988a87838c06056c89c0e8c4700a0452 F src/bitvec.c d1f21d7d91690747881f03940584f4cc548c9d3d F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 -F src/btree.c e807409602996359211c855078fa5cbcf59da032 +F src/btree.c 13f924763ebd6bd55e0b74d139ee363a42abbe0f F src/btree.h 00d4cdb747c4172a5566faf037116985dbbc377e F src/btreeInt.h 0c19847f87ab82e4f5e67750a069f10829475da6 F src/build.c e47b6fffe14a28d9050e6747beebb01597d37542 @@ -324,7 +324,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 388c023582b17890f10c980b30ec1922b471753b F src/os_win.c 40b3af7a47eb1107d0d69e592bec345a3b7b798a F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c a952e4b4c2e85e1588df5972688de9b6c989dbdd +F src/pager.c 57e5775ba3d8059e37e760f8a6f064079196eb9d F src/pager.h 1e7b0fc3846b71bd95b4b3300820d756895cb4ef F src/parse.y e9accdb2cb1795f75f478e7ce89e17b19d7d4da7 F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0 @@ -361,7 +361,7 @@ F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12 F src/test_backup.c 2e6e6a081870150f20c526a2e9d0d29cda47d803 F src/test_blob.c e5a7a81d61a780da79101aeb1e60d300af169e07 F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f -F src/test_config.c fb2e5d354d9a077f5fbb261652eff4787deb104f +F src/test_config.c 8870dbb8809e2bf7f766828934db7d70c907f346 F src/test_demovfs.c 0de72c2c89551629f58486fde5734b7d90758852 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f @@ -404,14 +404,14 @@ F src/vdbe.c 97b07a1af65971dab3d326742e912ac8c12108dd F src/vdbe.h 7a75045d879118b9d3af7e8b3c108f2f27c51473 F src/vdbeInt.h 8b54e01ad0463590e7cffabce0bc36da9ee4f816 F src/vdbeapi.c bda74ef4b5103d7b4a4be36f936d3cf2b56a7d6f -F src/vdbeaux.c 4fae90f55736b7537d0fcd2afe52849edbbb7ecd +F src/vdbeaux.c 5a07e354850fcc96c535f9b0b6ded4c705483a01 F src/vdbeblob.c 4f2e8e075d238392df98c5e03a64342465b03f90 F src/vdbemem.c ae38a0d35ae71cf604381a887c170466ba518090 F src/vdbesort.c f5009e7a35e3065635d8918b9a31f498a499976b F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 F src/vtab.c d31174e4c8f592febab3fa7f69e18320b4fd657a F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb -F src/wal.c b8811f666d785820437d95386c12b85951418c50 +F src/wal.c 7b8c8d1cb09f128d014a1677bc3d3935f729a2f9 F src/wal.h 903ef67e17f8b466dc7cfc4186fc23e80be10ff8 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba F src/where.c 66518a14a1238611aa0744d6980b6b7f544f4816 @@ -1227,9 +1227,9 @@ F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unique2.test 41e7f83c6827605991160a31380148a9fc5f1339 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 -F test/unlocked.test d143a871bd874311d4e5c98ce458218ca6b579fd -F test/unlocked2.test aaa42a08052a146466220b6c3a062ff3c4776ad6 -F test/unlocked3.test 2cacc31742561f6924706ed9472d0095839da9e4 +F test/unlocked.test 51be7f150c8d3bad50b22693e87a91a628e654be +F test/unlocked2.test 2d969d1b4c3e832169d0c97f522fa5955877785a +F test/unlocked3.test 6c99bc0c3f19ad8ca8c1632b09b089164718dc35 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 876810c28b3ad573ae46050ec699ef7eea4e313d -R 11d77b8f4d3f6d9b5ad54020bb4092ee +P 701302b4bd62ca7aefe643eac096a0ee672a62fa +R 736937b7bbb824036427389285348275 U dan -Z 46d9657b2162cb9e544c380eb188ec18 +Z a958ab41b63a16e8331a9ac18c76c8a0 diff --git a/manifest.uuid b/manifest.uuid index 93c1197630..1f1213c7c1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -701302b4bd62ca7aefe643eac096a0ee672a62fa \ No newline at end of file +041135575417201bbcf0544cc69dcb7369c7fb34 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 0b6b1305ae..8a99dbde14 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3346,9 +3346,11 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ if( rc==SQLITE_OK ){ rc = newDatabase(pBt); } +#ifdef SQLITE_ENABLE_UNLOCKED if( rc==SQLITE_OK && sqlite3PagerIsUnlocked(pBt->pPager) ){ rc = btreePtrmapAllocate(pBt); } +#endif } } diff --git a/src/pager.c b/src/pager.c index c2344e3747..9942211edb 100644 --- a/src/pager.c +++ b/src/pager.c @@ -1746,9 +1746,11 @@ static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ */ static void pagerFreeBitvecs(Pager *pPager){ sqlite3BitvecDestroy(pPager->pInJournal); - sqlite3BitvecDestroy(pPager->pAllRead); pPager->pInJournal = 0; +#ifdef SQLITE_ENABLE_UNLOCKED + sqlite3BitvecDestroy(pPager->pAllRead); pPager->pAllRead = 0; +#endif } /* @@ -3021,7 +3023,6 @@ static int pagerUndoCallback(void *pCtx, Pgno iPg){ static int pagerRollbackWal(Pager *pPager){ int rc; /* Return Code */ PgHdr *pList; /* List of dirty pages to revert */ - int bPage1 = 0; /* True if page 1 has been undone */ /* For all pages in the cache that are currently dirty or have already ** been written (but not committed) to the log file, do one of the @@ -3033,20 +3034,22 @@ static int pagerRollbackWal(Pager *pPager){ pPager->dbSize = pPager->dbOrigSize; rc = sqlite3WalUndo(pPager->pWal, pagerUndoCallback, (void *)pPager); pList = sqlite3PcacheDirtyList(pPager->pPCache); + + /* If this is an UNLOCKED transaction, then page 1 must be reread from + ** the db file, even if it is not dirty. This is because the b-tree layer + ** may have already zeroed the nFree and iTrunk header fields. */ +#ifdef SQLITE_ENABLE_UNLOCKED + if( rc==SQLITE_OK && (pList==0 || pList->pgno!=1) && pPager->pAllRead ){ + rc = pagerUndoCallback((void*)pPager, 1); + } +#endif + while( pList && rc==SQLITE_OK ){ PgHdr *pNext = pList->pDirty; - if( pList->pgno==1 ) bPage1 = 1; rc = pagerUndoCallback((void *)pPager, pList->pgno); pList = pNext; } - /* If this is an UNLOCKED transaction, then page 1 must be reread from the - ** db file, even if it is not dirty. This is because the b-tree layer may - ** have already zeroed the nFree and iTrunk header fields. */ - if( rc==SQLITE_OK && bPage1==0 && pPager->pAllRead ){ - rc = pagerUndoCallback((void*)pPager, 1); - } - return rc; } @@ -3086,7 +3089,6 @@ static int pagerWalFrames( ** list here. */ PgHdr **ppNext = &pList; nList = 0; - for(p=pList; (*ppNext = p)!=0; p=p->pDirty){ if( p->pgno<=nTruncate ){ ppNext = &p->pDirty; @@ -4452,7 +4454,9 @@ static int pagerStress(void *p, PgHdr *pPg){ if( pagerUseWal(pPager) ){ /* If the transaction is a "BEGIN UNLOCKED" transaction, the page ** cannot be flushed to disk. Return early in this case. */ +#ifdef SQLITE_ENABLE_UNLOCKED if( pPager->pAllRead ) return SQLITE_OK; +#endif /* Write a single frame for this page to the log. */ rc = subjournalPageIfRequired(pPg); @@ -5297,10 +5301,12 @@ int sqlite3PagerAcquire( /* If this is an UNLOCKED transaction and the page being read was ** present in the database file when the transaction was opened, ** mark it as read in the pAllRead vector. */ +#ifdef SQLITE_ENABLE_UNLOCKED if( pPager->pAllRead && pgno<=pPager->dbOrigSize ){ rc = sqlite3BitvecSet(pPager->pAllRead, pgno); if( rc!=SQLITE_OK ) goto pager_acquire_err; } +#endif /* If the pager is in the error state, return an error immediately. ** Otherwise, request the page from the PCache layer. */ @@ -5606,7 +5612,9 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ if( ALWAYS(pPager->eState==PAGER_READER) ){ assert( pPager->pInJournal==0 ); +#ifdef SQLITE_ENABLE_UNLOCKED assert( pPager->pAllRead==0 ); +#endif if( pagerUseWal(pPager) ){ /* If the pager is configured to use locking_mode=exclusive, and an @@ -5625,13 +5633,16 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ ** The busy-handler is not invoked if another connection already ** holds the write-lock. If possible, the upper layer will call it. */ - if( exFlag>=0 ){ - rc = sqlite3WalBeginWriteTransaction(pPager->pWal); - }else{ +#ifdef SQLITE_ENABLE_UNLOCKED + if( exFlag<0 ){ pPager->pAllRead = sqlite3BitvecCreate(pPager->dbSize); if( pPager->pAllRead==0 ){ rc = SQLITE_NOMEM; } + }else +#endif + { + rc = sqlite3WalBeginWriteTransaction(pPager->pWal); } }else{ /* Obtain a RESERVED lock on the database file. If the exFlag parameter @@ -5938,9 +5949,11 @@ int sqlite3PagerWrite(PgHdr *pPg){ ** to sqlite3PagerWrite(). In other words, return TRUE if it is ok ** to change the content of the page. */ +#if defined(SQLITE_ENABLE_UNLOCKED) || !defined(NDEBUG) int sqlite3PagerIswriteable(DbPage *pPg){ return pPg->flags & PGHDR_WRITEABLE; } +#endif /* ** A call to this routine tells the pager that it is not necessary to @@ -6088,15 +6101,24 @@ int sqlite3PagerSync(Pager *pPager, const char *zMaster){ } /* -** This function may only be called while a write-transaction is active in -** rollback. If the connection is in WAL mode, this call is a no-op. -** Otherwise, if the connection does not already have an EXCLUSIVE lock on -** the database file, an attempt is made to obtain one. +** This function is called to ensure that all locks required to commit the +** current write-transaction to the database file are held. If the db is +** in rollback mode, this means the EXCLUSIVE lock on the database file. ** -** If the EXCLUSIVE lock is already held or the attempt to obtain it is -** successful, or the connection is in WAL mode, SQLITE_OK is returned. -** Otherwise, either SQLITE_BUSY or an SQLITE_IOERR_XXX error code is -** returned. +** Or, if this is a non-UNLOCKED transaction on a wal-mode database, this +** function is a no-op. +** +** If this is an UNLOCKED transaction on a wal-mode database, this function +** attempts to obtain the WRITER lock on the wal file and also checks to +** see that the transaction can be safely committed (does not commit with +** any other transaction committed since it was opened). +** +** If the required locks are already held or successfully obtained and +** the transaction can be committed, SQLITE_OK is returned. If a required lock +** cannot be obtained, SQLITE_BUSY is returned. Or, if the current transaction +** is UNLOCKED and cannot be committed due to a conflict, SQLITE_BUSY_SNAPSHOT +** is returned. Otherwise, if some other error occurs (IO error, OOM etc.), +** and SQLite error code is returned. */ int sqlite3PagerExclusiveLock(Pager *pPager, PgHdr *pPage1){ int rc = SQLITE_OK; @@ -6107,23 +6129,38 @@ int sqlite3PagerExclusiveLock(Pager *pPager, PgHdr *pPage1){ assert( assert_pager_state(pPager) ); if( 0==pagerUseWal(pPager) ){ rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); - }else{ + } +#ifdef SQLITE_ENABLE_UNLOCKED + else{ if( pPager->pAllRead ){ /* This is an UNLOCKED transaction. Attempt to lock the wal database ** here. If SQLITE_BUSY (but not SQLITE_BUSY_SNAPSHOT) is returned, ** invoke the busy-handler and try again for as long as it returns ** non-zero. */ do { - /* rc = sqlite3WalBeginWriteTransaction(pWal); */ rc = sqlite3WalLockForCommit(pPager->pWal, pPage1, pPager->pAllRead); }while( rc==SQLITE_BUSY && pPager->xBusyHandler(pPager->pBusyHandlerArg) ); } } +#endif return rc; } +#ifdef SQLITE_ENABLE_UNLOCKED +/* +** This function is called as part of committing an UNLOCKED transaction. +** At this point the wal WRITER lock is held, and all pages in the cache +** except for page 1 are compatible with the snapshot at the head of the +** wal file. +** +** This function updates the in-memory data structures and reloads the +** contents of page 1 so that the client is operating on the snapshot +** at the head of the wal file. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ int sqlite3PagerUpgradeSnapshot(Pager *pPager, DbPage *pPage1){ int rc; u32 iFrame = 0; @@ -6140,8 +6177,18 @@ int sqlite3PagerUpgradeSnapshot(Pager *pPager, DbPage *pPage1){ return rc; } -void sqlite3PagerSetDbsize(Pager *pPager, Pgno nFinal){ - pPager->dbSize = nFinal; +/* +** Set the in-memory cache of the database file size to nSz pages. +*/ +void sqlite3PagerSetDbsize(Pager *pPager, Pgno nSz){ + pPager->dbSize = nSz; +} + +/* +** Return true if this pager is currently within an UNLOCKED transaction. +*/ +int sqlite3PagerIsUnlocked(Pager *pPager){ + return pPager->pAllRead!=0; } /* @@ -6153,13 +6200,7 @@ void sqlite3PagerDropExclusiveLock(Pager *pPager){ sqlite3WalEndWriteTransaction(pPager->pWal); } } - -/* -** Return true if this pager is currently within an UNLOCKED transaction. -*/ -int sqlite3PagerIsUnlocked(Pager *pPager){ - return pPager->pAllRead!=0; -} +#endif /* ifdef SQLITE_ENABLE_UNLOCKED */ /* diff --git a/src/test_config.c b/src/test_config.c index 0aa29c70d7..4d2dece83a 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -573,6 +573,12 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY); Tcl_SetVar2(interp, "sqlite_options", "truncate_opt", "1", TCL_GLOBAL_ONLY); #endif +#ifdef SQLITE_ENABLE_UNLOCKED + Tcl_SetVar2(interp, "sqlite_options", "unlocked", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "unlocked", "0", TCL_GLOBAL_ONLY); +#endif + #ifdef SQLITE_OMIT_UTF16 Tcl_SetVar2(interp, "sqlite_options", "utf16", "0", TCL_GLOBAL_ONLY); #else diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 354deb7294..ea467e08a3 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2025,6 +2025,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ } } +#ifdef SQLITE_ENABLE_UNLOCKED if( db->bUnlocked && (rc & 0xFF)==SQLITE_BUSY ){ /* An SQLITE_BUSY or SQLITE_BUSY_SNAPSHOT was encountered while ** attempting to take the WRITER lock on a wal file. Release the @@ -2038,6 +2039,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ } } } +#endif if( rc!=SQLITE_OK ){ return rc; diff --git a/src/wal.c b/src/wal.c index d43f939f00..033d33b083 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2566,12 +2566,60 @@ int sqlite3WalBeginWriteTransaction(Wal *pWal){ return rc; } -/* -** TODO: Combine some code with BeginWriteTransaction() +/* +** This function is called by a writer that has a read-lock on aReadmark[0] +** (pWal->readLock==0). This function relinquishes that lock and takes a +** lock on a different aReadmark[] slot. ** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +static int walUpgradeReadlock(Wal *pWal){ + int cnt; + int rc; + assert( pWal->writeLock && pWal->readLock==0 ); + walUnlockShared(pWal, WAL_READ_LOCK(0)); + pWal->readLock = -1; + cnt = 0; + do{ + int notUsed; + rc = walTryBeginRead(pWal, ¬Used, 1, ++cnt); + }while( rc==WAL_RETRY ); + assert( (rc&0xff)!=SQLITE_BUSY ); /* BUSY not possible when useWal==1 */ + testcase( (rc&0xff)==SQLITE_IOERR ); + testcase( rc==SQLITE_PROTOCOL ); + testcase( rc==SQLITE_OK ); + return rc; +} + + +#ifdef SQLITE_ENABLE_UNLOCKED +/* ** This function is only ever called when committing a "BEGIN UNLOCKED" ** transaction. It may be assumed that no frames have been written to -** the wal file. +** the wal file. The second parameter is a pointer to the in-memory +** representation of page 1 of the database (which may or may not be +** dirty). The third is a bitvec with a bit set for each page in the +** database file that was read by the current unlocked transaction. +** +** This function performs three tasks: +** +** 1) It obtains the WRITER lock on the wal file, +** +** 2) It checks that there are no conflicts between the current +** transaction and any transactions committed to the wal file since +** it was opened, and +** +** 3) It ejects any non-dirty pages from the page-cache that have been +** written by another client since the UNLOCKED transaction was started +** (so as to avoid ending up with an inconsistent cache after the +** current transaction is committed). +** +** If no error occurs and the caller may proceed with committing the +** transaction, SQLITE_OK is returned. SQLITE_BUSY is returned if the WRITER +** lock cannot be obtained. Or, if the WRITER lock can be obtained but there +** are conflicts with a committed transaction, SQLITE_BUSY_SNAPSHOT. Finally, +** if an error (i.e. an OOM condition or IO error), an SQLite error code +** is returned. */ int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPage1, Bitvec *pAllRead){ Pager *pPager = pPage1->pPager; @@ -2668,30 +2716,17 @@ int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPage1, Bitvec *pAllRead){ } /* -** This function is called by a writer that has a read-lock on aReadmark[0] -** (pWal->readLock==0). This function relinquishes that lock and takes a -** lock on a different aReadmark[] slot. +** This function is called as part of committing an UNLOCKED transaction. +** It is assumed that sqlite3WalLockForCommit() has already been successfully +** called and so (a) the WRITER lock is held and (b) it is known that the +** wal-index-header stored in shared memory is not corrupt. +** +** Before returning, this function upgrades the client so that it is +** operating on the database snapshot currently at the head of the wal file +** (even if the UNLOCKED transaction ran against an older snapshot). ** ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ -static int walUpgradeReadlock(Wal *pWal){ - int cnt; - int rc; - assert( pWal->writeLock && pWal->readLock==0 ); - walUnlockShared(pWal, WAL_READ_LOCK(0)); - pWal->readLock = -1; - cnt = 0; - do{ - int notUsed; - rc = walTryBeginRead(pWal, ¬Used, 1, ++cnt); - }while( rc==WAL_RETRY ); - assert( (rc&0xff)!=SQLITE_BUSY ); /* BUSY not possible when useWal==1 */ - testcase( (rc&0xff)==SQLITE_IOERR ); - testcase( rc==SQLITE_PROTOCOL ); - testcase( rc==SQLITE_OK ); - return rc; -} - int sqlite3WalUpgradeSnapshot(Wal *pWal){ int rc = SQLITE_OK; assert( pWal->writeLock ); @@ -2706,6 +2741,7 @@ int sqlite3WalUpgradeSnapshot(Wal *pWal){ } return rc; } +#endif /* SQLITE_ENABLE_UNLOCKED */ /* ** End a write transaction. The commit has already been done. This diff --git a/test/unlocked.test b/test/unlocked.test index c743cc6328..5cbb31dbb9 100644 --- a/test/unlocked.test +++ b/test/unlocked.test @@ -15,6 +15,10 @@ source $testdir/tester.tcl source $testdir/lock_common.tcl set ::testprefix unlocked +ifcapable !unlocked { + finish_test + return +} do_execsql_test 1.0 { PRAGMA journal_mode = wal; diff --git a/test/unlocked2.test b/test/unlocked2.test index 6953753643..f28d62e0a1 100644 --- a/test/unlocked2.test +++ b/test/unlocked2.test @@ -15,6 +15,10 @@ source $testdir/tester.tcl source $testdir/lock_common.tcl set ::testprefix unlocked2 +ifcapable !unlocked { + finish_test + return +} do_multiclient_test tn { diff --git a/test/unlocked3.test b/test/unlocked3.test index 61e71c87c3..36299db381 100644 --- a/test/unlocked3.test +++ b/test/unlocked3.test @@ -19,6 +19,10 @@ source $testdir/lock_common.tcl set ::testprefix unlocked3 if {$AUTOVACUUM} { finish_test ; return } +ifcapable !unlocked { + finish_test + return +} proc create_schema {} { db eval { From f5e89dba9d70a4e1b94b8b7b6a519afae839d285 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 24 Aug 2015 19:08:10 +0000 Subject: [PATCH 016/179] Fix handling of attempts to modify the database schema, application_id or user_version within an UNLOCKED transaction. FossilOrigin-Name: 5b9f272113d21fd606903509d6f830fe60fac039 --- manifest | 16 +++++----- manifest.uuid | 2 +- src/vdbe.c | 34 ++++++++++++++------ src/wal.c | 79 ++++++++++++++++++++++++++++------------------ test/unlocked.test | 45 ++++++++++++++++++++++++-- 5 files changed, 124 insertions(+), 52 deletions(-) diff --git a/manifest b/manifest index 7e91525501..c33005f21e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompilation\swithout\sSQLITE_ENABLE_UNLOCKED.\sAlso\sother\scode\sorganization\sissues. -D 2015-08-24T16:00:08.023 +C Fix\shandling\sof\sattempts\sto\smodify\sthe\sdatabase\sschema,\sapplication_id\sor\suser_version\swithin\san\sUNLOCKED\stransaction. +D 2015-08-24T19:08:10.037 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -400,7 +400,7 @@ F src/update.c 487747b328b7216bb7f6af0695d6937d5c9e605f F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c bc9dd64b5db544218b871b66243871c202b2781f F src/vacuum.c d35c7291e94a470ee47695c22559a69aef9fdda1 -F src/vdbe.c 97b07a1af65971dab3d326742e912ac8c12108dd +F src/vdbe.c cb555e4a802ec96e16f48c54b57bf15b94616567 F src/vdbe.h 7a75045d879118b9d3af7e8b3c108f2f27c51473 F src/vdbeInt.h 8b54e01ad0463590e7cffabce0bc36da9ee4f816 F src/vdbeapi.c bda74ef4b5103d7b4a4be36f936d3cf2b56a7d6f @@ -411,7 +411,7 @@ F src/vdbesort.c f5009e7a35e3065635d8918b9a31f498a499976b F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 F src/vtab.c d31174e4c8f592febab3fa7f69e18320b4fd657a F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb -F src/wal.c 7b8c8d1cb09f128d014a1677bc3d3935f729a2f9 +F src/wal.c fd9a3b9fa79042377f3e5e42a91fd6e0daf1037e F src/wal.h 903ef67e17f8b466dc7cfc4186fc23e80be10ff8 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba F src/where.c 66518a14a1238611aa0744d6980b6b7f544f4816 @@ -1227,7 +1227,7 @@ F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unique2.test 41e7f83c6827605991160a31380148a9fc5f1339 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 -F test/unlocked.test 51be7f150c8d3bad50b22693e87a91a628e654be +F test/unlocked.test fecdc8d76e3495664c74961e6cd1aef9b59439c1 F test/unlocked2.test 2d969d1b4c3e832169d0c97f522fa5955877785a F test/unlocked3.test 6c99bc0c3f19ad8ca8c1632b09b089164718dc35 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 701302b4bd62ca7aefe643eac096a0ee672a62fa -R 736937b7bbb824036427389285348275 +P 041135575417201bbcf0544cc69dcb7369c7fb34 +R c0f10d85f21cf03acdf9208c7581f1d4 U dan -Z a958ab41b63a16e8331a9ac18c76c8a0 +Z 17437f740072ebd0dda6cab278d01c48 diff --git a/manifest.uuid b/manifest.uuid index 1f1213c7c1..e52ba34ef9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -041135575417201bbcf0544cc69dcb7369c7fb34 \ No newline at end of file +5b9f272113d21fd606903509d6f830fe60fac039 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index b63194655c..3cdb252fdf 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3227,19 +3227,24 @@ case OP_SetCookie: { /* in3 */ assert( sqlite3SchemaMutexHeld(db, pOp->p1, 0) ); pIn3 = &aMem[pOp->p3]; sqlite3VdbeMemIntegerify(pIn3); +#ifdef SQLITE_ENABLE_UNLOCKED + if( db->bUnlocked + && (pOp->p2==BTREE_USER_VERSION || pOp->p2==BTREE_APPLICATION_ID) + ){ + rc = SQLITE_ERROR; + sqlite3VdbeError(p, "cannot modify %s within UNLOCKED transaction", + pOp->p2==BTREE_USER_VERSION ? "user_version" : "application_id" + ); + break; + } +#endif /* See note about index shifting on OP_ReadCookie */ rc = sqlite3BtreeUpdateMeta(pDb->pBt, pOp->p2, (int)pIn3->u.i); if( pOp->p2==BTREE_SCHEMA_VERSION ){ - if( db->bUnlocked ){ - sqlite3VdbeError(p, "cannot modify database schema - " - "UNLOCKED transaction" - ); - rc = SQLITE_ERROR; - }else{ - /* When the schema cookie changes, record the new cookie internally */ - pDb->pSchema->schema_cookie = (int)pIn3->u.i; - db->flags |= SQLITE_InternChanges; - } + /* When the schema cookie changes, record the new cookie internally */ + assert( db->bUnlocked==0 ); + pDb->pSchema->schema_cookie = (int)pIn3->u.i; + db->flags |= SQLITE_InternChanges; }else if( pOp->p2==BTREE_FILE_FORMAT ){ /* Record changes in the file format */ pDb->pSchema->file_format = (u8)pIn3->u.i; @@ -6110,6 +6115,15 @@ case OP_Expire: { */ case OP_TableLock: { u8 isWriteLock = (u8)pOp->p3; +#ifdef SQLITE_ENABLE_UNLOCKED + if( isWriteLock && db->bUnlocked && pOp->p2==1 ){ + rc = SQLITE_ERROR; + sqlite3VdbeError(p, + "cannot modify database schema within UNLOCKED transaction"); + rc = SQLITE_ERROR; + break; + } +#endif if( isWriteLock || 0==(db->flags&SQLITE_ReadUncommitted) ){ int p1 = pOp->p1; assert( p1>=0 && p1nDb ); diff --git a/src/wal.c b/src/wal.c index 033d33b083..9a2152424a 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1950,25 +1950,14 @@ int sqlite3WalClose( } /* -** Try to read the wal-index header. Return 0 on success and 1 if -** there is a problem. -** -** The wal-index is in shared memory. Another thread or process might -** be writing the header at the same time this procedure is trying to -** read it, which might result in inconsistency. A dirty read is detected -** by verifying that both copies of the header are the same and also by -** a checksum on the header. -** -** If and only if the read is consistent and the header is different from -** pWal->hdr, then pWal->hdr is updated to the content of the new header -** and *pChanged is set to 1. -** -** If the checksum cannot be verified return non-zero. If the header -** is read successfully and the checksum verified, return zero. +** Try to copy the wal-index header from shared-memory into (*pHdr). Return +** zero if successful or non-zero otherwise. If the header is corrupted +** (either because the two copies are inconsistent or because the checksum +** values are incorrect), the read fails and non-zero is returned. */ -static int walIndexTryHdr(Wal *pWal, int *pChanged){ +static int walIndexLoadHdr(Wal *pWal, WalIndexHdr *pHdr){ u32 aCksum[2]; /* Checksum on the header content */ - WalIndexHdr h1, h2; /* Two copies of the header content */ + WalIndexHdr h2; /* Second copy of the header content */ WalIndexHdr volatile *aHdr; /* Header in shared memory */ /* The first page of the wal-index must be mapped at this point. */ @@ -1985,21 +1974,48 @@ static int walIndexTryHdr(Wal *pWal, int *pChanged){ ** reordering the reads and writes. */ aHdr = walIndexHdr(pWal); - memcpy(&h1, (void *)&aHdr[0], sizeof(h1)); + memcpy(pHdr, (void *)&aHdr[0], sizeof(h2)); walShmBarrier(pWal); memcpy(&h2, (void *)&aHdr[1], sizeof(h2)); - if( memcmp(&h1, &h2, sizeof(h1))!=0 ){ + if( memcmp(&h2, pHdr, sizeof(h2))!=0 ){ return 1; /* Dirty read */ } - if( h1.isInit==0 ){ + if( h2.isInit==0 ){ return 1; /* Malformed header - probably all zeros */ } - walChecksumBytes(1, (u8*)&h1, sizeof(h1)-sizeof(h1.aCksum), 0, aCksum); - if( aCksum[0]!=h1.aCksum[0] || aCksum[1]!=h1.aCksum[1] ){ + walChecksumBytes(1, (u8*)&h2, sizeof(h2)-sizeof(h2.aCksum), 0, aCksum); + if( aCksum[0]!=h2.aCksum[0] || aCksum[1]!=h2.aCksum[1] ){ return 1; /* Checksum does not match */ } + return 0; +} + +/* +** Try to read the wal-index header. Return 0 on success and 1 if +** there is a problem. +** +** The wal-index is in shared memory. Another thread or process might +** be writing the header at the same time this procedure is trying to +** read it, which might result in inconsistency. A dirty read is detected +** by verifying that both copies of the header are the same and also by +** a checksum on the header. +** +** If and only if the read is consistent and the header is different from +** pWal->hdr, then pWal->hdr is updated to the content of the new header +** and *pChanged is set to 1. +** +** If the checksum cannot be verified return non-zero. If the header +** is read successfully and the checksum verified, return zero. +*/ +static int walIndexTryHdr(Wal *pWal, int *pChanged){ + WalIndexHdr h1; /* Copy of the header content */ + + if( walIndexLoadHdr(pWal, &h1) ){ + return 1; + } + if( memcmp(&pWal->hdr, &h1, sizeof(WalIndexHdr)) ){ *pChanged = 1; memcpy(&pWal->hdr, &h1, sizeof(WalIndexHdr)); @@ -2636,16 +2652,19 @@ int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPage1, Bitvec *pAllRead){ ** transaction was started. */ if( rc==SQLITE_OK ){ - volatile WalIndexHdr *pHead; /* Head of the wal file */ - pHead = walIndexHdr(pWal); + WalIndexHdr head; - /* TODO: Check header checksum is good here. */ - - if( memcmp(&pWal->hdr, (void*)pHead, sizeof(WalIndexHdr))!=0 ){ + if( walIndexLoadHdr(pWal, &head) ){ + /* This branch is taken if the wal-index header is corrupted. This + ** occurs if some other writer has crashed while committing a + ** transaction to this database since the current unlocked transaction + ** was opened. */ + rc = SQLITE_BUSY_SNAPSHOT; + }else if( memcmp(&pWal->hdr, (void*)&head, sizeof(WalIndexHdr))!=0 ){ int iHash; - int iLastHash = walFramePage(pHead->mxFrame); + int iLastHash = walFramePage(head.mxFrame); u32 iFirst = pWal->hdr.mxFrame+1; /* First wal frame to check */ - if( memcmp(pWal->hdr.aSalt, (u32*)pHead->aSalt, sizeof(u32)*2) ){ + if( memcmp(pWal->hdr.aSalt, (u32*)head.aSalt, sizeof(u32)*2) ){ assert( pWal->readLock==0 ); iFirst = 1; } @@ -2660,7 +2679,7 @@ int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPage1, Bitvec *pAllRead){ int iMin = (iFirst - iZero); int iMax = (iHash==0) ? HASHTABLE_NPAGE_ONE : HASHTABLE_NPAGE; if( iMin<1 ) iMin = 1; - if( iMax>pHead->mxFrame ) iMax = pHead->mxFrame; + if( iMax>head.mxFrame ) iMax = head.mxFrame; for(i=iMin; i<=iMax; i++){ PgHdr *pPg; if( aPgno[i]==1 ){ diff --git a/test/unlocked.test b/test/unlocked.test index 5cbb31dbb9..0e22b97fa1 100644 --- a/test/unlocked.test +++ b/test/unlocked.test @@ -110,13 +110,21 @@ foreach {tn trans commit_ok} { foreach {tn sql} { 1 { CREATE TABLE xx(a, b) } 2 { DROP TABLE t1 } + 3 { CREATE INDEX i1 ON t1(a) } + 4 { CREATE VIEW v1 AS SELECT * FROM t1 } + 5 { CREATE TEMP TABLE xx(a, b) } } { do_catchsql_test 1.7.$tn.1 " BEGIN UNLOCKED; $sql - " {1 {cannot modify database schema - UNLOCKED transaction}} + " {1 {cannot modify database schema within UNLOCKED transaction}} - do_execsql_test 1.7.$tn.2 ROLLBACK + do_execsql_test 1.7.$tn.2 { + SELECT sql FROM sqlite_master; + SELECT sql FROM sqlite_temp_master; + } {{CREATE TABLE t1(a, b)}} + + do_execsql_test 1.7.$tn.3 COMMIT } #------------------------------------------------------------------------- @@ -408,6 +416,37 @@ do_multiclient_test tn { do_test 2.$tn.7.4 { sql3 { PRAGMA integrity_check } } ok } - +#------------------------------------------------------------------------- +# Unlocked transactions may not modify the user_version or application_id. +# +reset_db +do_execsql_test 3.0 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES('a', 'b'); + PRAGMA user_version = 10; +} {wal} +do_execsql_test 3.1 { + BEGIN UNLOCKED; + INSERT INTO t1 VALUES('c', 'd'); + SELECT * FROM t1; +} {a b c d} +do_catchsql_test 3.2 { + PRAGMA user_version = 11; +} {1 {cannot modify user_version within UNLOCKED transaction}} +do_execsql_test 3.3 { + PRAGMA user_version; + SELECT * FROM t1; +} {10 a b c d} +do_catchsql_test 3.4 { + PRAGMA application_id = 11; +} {1 {cannot modify application_id within UNLOCKED transaction}} +do_execsql_test 3.5 { + COMMIT; + PRAGMA user_version; + PRAGMA application_id; + SELECT * FROM t1; +} {10 0 a b c d} finish_test + From bf3cf57e15a16805f5dfa200c3bfa800d7b6bb2d Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 24 Aug 2015 19:56:04 +0000 Subject: [PATCH 017/179] Change "BEGIN UNLOCKED" to "BEGIN CONCURRENT". FossilOrigin-Name: ba1ab858e2997c88dd7eee6e6893a8616d85c665 --- manifest | 42 ++++++------- manifest.uuid | 2 +- src/btree.c | 46 +++++++------- src/btreeInt.h | 10 +-- src/build.c | 2 +- src/pager.c | 46 +++++++------- src/pager.h | 2 +- src/parse.y | 2 +- src/sqliteInt.h | 2 +- src/test_config.c | 6 +- src/vacuum.c | 2 +- src/vdbe.c | 34 +++++------ src/vdbeaux.c | 10 +-- src/wal.c | 24 ++++---- test/{unlocked.test => concurrent.test} | 74 +++++++++++------------ test/{unlocked2.test => concurrent2.test} | 44 +++++++------- test/{unlocked3.test => concurrent3.test} | 8 +-- tool/mkkeywordhash.c | 2 +- 18 files changed, 179 insertions(+), 179 deletions(-) rename test/{unlocked.test => concurrent.test} (85%) rename test/{unlocked2.test => concurrent2.test} (87%) rename test/{unlocked3.test => concurrent3.test} (93%) diff --git a/manifest b/manifest index c33005f21e..ceae322d8a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\shandling\sof\sattempts\sto\smodify\sthe\sdatabase\sschema,\sapplication_id\sor\suser_version\swithin\san\sUNLOCKED\stransaction. -D 2015-08-24T19:08:10.037 +C Change\s"BEGIN\sUNLOCKED"\sto\s"BEGIN\sCONCURRENT". +D 2015-08-24T19:56:04.567 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -279,10 +279,10 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 4d9134dc988a87838c06056c89c0e8c4700a0452 F src/bitvec.c d1f21d7d91690747881f03940584f4cc548c9d3d F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 -F src/btree.c 13f924763ebd6bd55e0b74d139ee363a42abbe0f +F src/btree.c f4db20fab7bf5ebd9f12ef7e3a925ca3c2c34fa8 F src/btree.h 00d4cdb747c4172a5566faf037116985dbbc377e -F src/btreeInt.h 0c19847f87ab82e4f5e67750a069f10829475da6 -F src/build.c e47b6fffe14a28d9050e6747beebb01597d37542 +F src/btreeInt.h 171864bcd81635583dab7b8a04b19b454b18ef80 +F src/build.c 2890c89e0dc274171c18195ff2a6364be47c2b1d F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b @@ -324,9 +324,9 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 388c023582b17890f10c980b30ec1922b471753b F src/os_win.c 40b3af7a47eb1107d0d69e592bec345a3b7b798a F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 57e5775ba3d8059e37e760f8a6f064079196eb9d -F src/pager.h 1e7b0fc3846b71bd95b4b3300820d756895cb4ef -F src/parse.y e9accdb2cb1795f75f478e7ce89e17b19d7d4da7 +F src/pager.c 2e2559e64e825e39c033c0744237733cec70d636 +F src/pager.h 1335b624cd540815c8c977172589d208d1c251a6 +F src/parse.y dc3dda4b191ed54ae7d3662ffc9ee21ba91850ae F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0 F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9 F src/pcache1.c a3fe31b17e841ec70beee72a2c960e9c787a8857 @@ -342,7 +342,7 @@ F src/shell.c b1f91e60918df3a68efad1e3a11696b9a7e23d23 F src/sqlite.h.in 378bebc8fe6a88bade25e5f23b7e6123fdc64b00 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h f700e6a9dd1fdcccc9951ab022b366fb66b9e413 -F src/sqliteInt.h d4963f5f6b985b8f967be8e6e103be3cf36f1f42 +F src/sqliteInt.h 54c3393fc26737db60ed369881d91fa54b71987d F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e @@ -361,7 +361,7 @@ F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12 F src/test_backup.c 2e6e6a081870150f20c526a2e9d0d29cda47d803 F src/test_blob.c e5a7a81d61a780da79101aeb1e60d300af169e07 F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f -F src/test_config.c 8870dbb8809e2bf7f766828934db7d70c907f346 +F src/test_config.c f853203a1d2035370ea26ad48248f6d00a10b752 F src/test_demovfs.c 0de72c2c89551629f58486fde5734b7d90758852 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f @@ -399,19 +399,19 @@ F src/trigger.c 322f23aad694e8f31d384dcfa386d52a48d3c52f F src/update.c 487747b328b7216bb7f6af0695d6937d5c9e605f F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c bc9dd64b5db544218b871b66243871c202b2781f -F src/vacuum.c d35c7291e94a470ee47695c22559a69aef9fdda1 -F src/vdbe.c cb555e4a802ec96e16f48c54b57bf15b94616567 +F src/vacuum.c 983cc3754718ef169a6ea9aef86798bd28106f21 +F src/vdbe.c 84ac0688d7e1921d8e802f2e4e1a38273e069d0c F src/vdbe.h 7a75045d879118b9d3af7e8b3c108f2f27c51473 F src/vdbeInt.h 8b54e01ad0463590e7cffabce0bc36da9ee4f816 F src/vdbeapi.c bda74ef4b5103d7b4a4be36f936d3cf2b56a7d6f -F src/vdbeaux.c 5a07e354850fcc96c535f9b0b6ded4c705483a01 +F src/vdbeaux.c 9c59b0604489ebe62869a59f3b627c822d32b278 F src/vdbeblob.c 4f2e8e075d238392df98c5e03a64342465b03f90 F src/vdbemem.c ae38a0d35ae71cf604381a887c170466ba518090 F src/vdbesort.c f5009e7a35e3065635d8918b9a31f498a499976b F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 F src/vtab.c d31174e4c8f592febab3fa7f69e18320b4fd657a F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb -F src/wal.c fd9a3b9fa79042377f3e5e42a91fd6e0daf1037e +F src/wal.c a21412a803f0eafecf48e707fb97c64368a0d267 F src/wal.h 903ef67e17f8b466dc7cfc4186fc23e80be10ff8 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba F src/where.c 66518a14a1238611aa0744d6980b6b7f544f4816 @@ -523,6 +523,9 @@ F test/collate9.test 3adcc799229545940df2f25308dd1ad65869145a F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b +F test/concurrent.test 631e398b127a13ddfbcc8e64f438d9f82b261ffe w test/unlocked.test +F test/concurrent2.test 21a15630192ba92287070a1a58a6e497d393c55d w test/unlocked2.test +F test/concurrent3.test 7dcf81372c06cbac58e7e630aebf7292945947bb w test/unlocked3.test F test/conflict.test 841bcf7cabbfca39c577eb8411ea8601843b46a8 F test/conflict2.test 0d3af4fb534fa1bd020c79960bb56e4d52655f09 F test/conflict3.test dec0634c0f31dec9a4b01c63063e939f0cd21b6b @@ -1227,9 +1230,6 @@ F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unique2.test 41e7f83c6827605991160a31380148a9fc5f1339 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 -F test/unlocked.test fecdc8d76e3495664c74961e6cd1aef9b59439c1 -F test/unlocked2.test 2d969d1b4c3e832169d0c97f522fa5955877785a -F test/unlocked3.test 6c99bc0c3f19ad8ca8c1632b09b089164718dc35 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b @@ -1341,7 +1341,7 @@ F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh d1a2da0e15b2ed33d60af35c7e9d483f13a8eb9f -F tool/mkkeywordhash.c de4a823fe66f9e8c39c86e465ac7285fd6935bb8 +F tool/mkkeywordhash.c 8d78ea188240bc08ec080adf3af84717f013e69a F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkpragmatab.tcl 84af2b180484323a2ea22a2279e8bd9e3e1e492e F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 041135575417201bbcf0544cc69dcb7369c7fb34 -R c0f10d85f21cf03acdf9208c7581f1d4 +P 5b9f272113d21fd606903509d6f830fe60fac039 +R 7a2246f9f176c887e1e1da04647372c5 U dan -Z 17437f740072ebd0dda6cab278d01c48 +Z f0bce293b4f554fd07229d405fb3ff97 diff --git a/manifest.uuid b/manifest.uuid index e52ba34ef9..e856c7e395 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5b9f272113d21fd606903509d6f830fe60fac039 \ No newline at end of file +ba1ab858e2997c88dd7eee6e6893a8616d85c665 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 8a99dbde14..e391255a41 100644 --- a/src/btree.c +++ b/src/btree.c @@ -440,10 +440,10 @@ static void downgradeAllSharedCacheTableLocks(Btree *p){ #endif /* SQLITE_OMIT_SHARED_CACHE */ -#ifdef SQLITE_ENABLE_UNLOCKED +#ifdef SQLITE_ENABLE_CONCURRENT /* ** The following structure - BtreePtrmap - stores the in-memory pointer map -** used for newly allocated pages in UNLOCKED transactions. Such pages are +** used for newly allocated pages in CONCURRENT transactions. Such pages are ** always allocated in a contiguous block (from the end of the file) starting ** with page BtreePtrmap.iFirst. */ @@ -588,7 +588,7 @@ static void btreePtrmapEnd(BtShared *pBt, int op, int iSvpt){ } /* -** This function is called after an UNLOCKED transaction is opened on the +** This function is called after an CONCURRENT transaction is opened on the ** database. It allocates the BtreePtrmap structure used to track pointers ** to allocated pages and zeroes the nFree/iTrunk fields in the database ** header on page 1. @@ -596,7 +596,7 @@ static void btreePtrmapEnd(BtShared *pBt, int op, int iSvpt){ static int btreePtrmapAllocate(BtShared *pBt){ int rc = SQLITE_OK; BtreePtrmap *pMap = sqlite3_malloc(sizeof(BtreePtrmap)); - assert( pBt->pMap==0 && sqlite3PagerIsUnlocked(pBt->pPager) ); + assert( pBt->pMap==0 && sqlite3PagerIsConcurrent(pBt->pPager) ); if( pMap==0 ){ rc = SQLITE_NOMEM; }else{ @@ -1074,7 +1074,7 @@ static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){ /* The master-journal page number is never added to a pointer-map page */ assert( 0==PTRMAP_ISPAGE(pBt, PENDING_BYTE_PAGE(pBt)) ); -#ifdef SQLITE_ENABLE_UNLOCKED +#ifdef SQLITE_ENABLE_CONCURRENT if( pBt->pMap ){ *pRC = btreePtrmapStore(pBt, key, eType, parent); return; @@ -3339,15 +3339,15 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ if( (pBt->btsFlags & BTS_READ_ONLY)!=0 ){ rc = SQLITE_READONLY; }else{ - int exFlag = (p->db->bUnlocked && !ISAUTOVACUUM) ? -1 : (wrflag>1); + int exFlag = (p->db->bConcurrent && !ISAUTOVACUUM) ? -1 : (wrflag>1); int bSubjInMem = sqlite3TempInMemory(p->db); - assert( p->db->bUnlocked==0 || wrflag==1 ); + assert( p->db->bConcurrent==0 || wrflag==1 ); rc = sqlite3PagerBegin(pBt->pPager, exFlag, bSubjInMem); if( rc==SQLITE_OK ){ rc = newDatabase(pBt); } -#ifdef SQLITE_ENABLE_UNLOCKED - if( rc==SQLITE_OK && sqlite3PagerIsUnlocked(pBt->pPager) ){ +#ifdef SQLITE_ENABLE_CONCURRENT + if( rc==SQLITE_OK && sqlite3PagerIsConcurrent(pBt->pPager) ){ rc = btreePtrmapAllocate(pBt); } #endif @@ -3848,9 +3848,9 @@ static int autoVacuumCommit(BtShared *pBt){ # define setChildPtrmaps(x) SQLITE_OK #endif -#ifdef SQLITE_ENABLE_UNLOCKED +#ifdef SQLITE_ENABLE_CONCURRENT /* -** This function is called as part of merging an UNLOCKED transaction with +** This function is called as part of merging an CONCURRENT transaction with ** the snapshot at the head of the wal file. It relocates all pages in the ** range iFirst..iLast, inclusive. It is assumed that the BtreePtrmap ** structure at BtShared.pMap contains the location of the pointers to each @@ -3918,7 +3918,7 @@ static int btreeRelocateRange( /* ** The b-tree handle passed as the only argument is about to commit an -** UNLOCKED transaction. At this point it is guaranteed that this is +** CONCURRENT transaction. At this point it is guaranteed that this is ** possible - the wal WRITER lock is held and it is known that there are ** no conflicts with committed transactions. */ @@ -3937,7 +3937,7 @@ static int btreeFixUnlocked(Btree *p){ Pgno nPage = btreePagecount(pBt); u32 nFree = get4byte(&p1[36]); - assert( sqlite3PagerIsUnlocked(pPager) ); + assert( sqlite3PagerIsConcurrent(pPager) ); assert( pBt->pMap ); rc = sqlite3PagerUpgradeSnapshot(pPager, pPage1->pDbPage); assert( p1==pPage1->aData ); @@ -3967,7 +3967,7 @@ static int btreeFixUnlocked(Btree *p){ if( nHPage<(pMap->iFirst-1) ){ /* The database consisted of (pMap->iFirst-1) pages when the current - ** unlocked transaction was opened. And an unlocked transaction may + ** concurrent transaction was opened. And an concurrent transaction may ** not be executed on an auto-vacuum database - so the db should ** not have shrunk since the transaction was opened. Therefore nHPage ** should be set to (pMap->iFirst-1) or greater. */ @@ -4042,7 +4042,7 @@ int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zMaster){ #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ - assert( ISUNLOCKED==0 ); + assert( ISCONCURRENT==0 ); rc = autoVacuumCommit(pBt); if( rc!=SQLITE_OK ){ sqlite3BtreeLeave(p); @@ -4053,7 +4053,7 @@ int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zMaster){ sqlite3PagerTruncateImage(pBt->pPager, pBt->nPage); } #endif - if( rc==SQLITE_OK && ISUNLOCKED ){ + if( rc==SQLITE_OK && ISCONCURRENT ){ rc = btreeFixUnlocked(p); } if( rc==SQLITE_OK ){ @@ -4101,7 +4101,7 @@ static void btreeEndTransaction(Btree *p){ unlockBtreeIfUnused(pBt); } - /* If this was an UNLOCKED transaction, delete the pBt->pMap object */ + /* If this was an CONCURRENT transaction, delete the pBt->pMap object */ btreePtrmapDelete(pBt); btreeIntegrity(p); } @@ -5860,14 +5860,14 @@ static int allocateBtreePage( ** stores stores the total number of pages on the freelist. */ n = get4byte(&pPage1->aData[36]); testcase( n==mxPage-1 ); - if( ISUNLOCKED==0 && n>=mxPage ){ + if( ISCONCURRENT==0 && n>=mxPage ){ return SQLITE_CORRUPT_BKPT; } /* Ensure page 1 is writable. This function will either change the number ** of pages in the free-list or the size of the database file. Since both ** of these operations involve modifying page 1 header fields, page 1 - ** will definitely be written by this transaction. If this is an UNLOCKED + ** will definitely be written by this transaction. If this is an CONCURRENT ** transaction, ensure the BtreePtrmap structure has been allocated. */ rc = sqlite3PagerWrite(pPage1->pDbPage); if( rc ) return rc; @@ -5883,7 +5883,7 @@ static int allocateBtreePage( ** the entire-list will be searched for that page. */ if( eMode==BTALLOC_EXACT ){ - assert( ISAUTOVACUUM!=ISUNLOCKED ); + assert( ISAUTOVACUUM!=ISCONCURRENT ); if( ISAUTOVACUUM ){ if( nearby<=mxPage ){ u8 eType; @@ -9974,14 +9974,14 @@ int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); } ** current write-transaction to the database file are held. If the db is ** in rollback mode, this means the EXCLUSIVE lock on the database file. ** -** Or, if this is an UNLOCKED transaction on a wal-mode database, the WRITER +** Or, if this is an CONCURRENT transaction on a wal-mode database, the WRITER ** lock on the wal file. In this case this function also checks that the -** UNLOCKED transaction can be safely committed (does not commit with any +** CONCURRENT transaction can be safely committed (does not commit with any ** other transaction committed since it was opened). ** ** SQLITE_OK is returned if successful. SQLITE_BUSY if the required locks ** cannot be obtained due to a conflicting lock. If the locks cannot be -** obtained for an UNLOCKED transaction due to a conflict with an already +** obtained for an CONCURRENT transaction due to a conflict with an already ** committed transaction, SQLITE_BUSY_SNAPSHOT is returned. Otherwise, if ** some other error (OOM, IO, etc.) occurs, the relevant SQLite error code ** is returned. diff --git a/src/btreeInt.h b/src/btreeInt.h index 4c23629cc8..e686466e81 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -448,7 +448,7 @@ struct BtShared { Btree *pWriter; /* Btree with currently open write transaction */ #endif u8 *pTmpSpace; /* Temp space sufficient to hold a single cell */ -#ifdef SQLITE_ENABLE_UNLOCKED +#ifdef SQLITE_ENABLE_CONCURRENT BtreePtrmap *pMap; #endif }; @@ -663,13 +663,13 @@ struct BtCursor { #define ISAUTOVACUUM 0 #endif -#ifdef SQLITE_ENABLE_UNLOCKED -# define ISUNLOCKED (pBt->pMap!=0) +#ifdef SQLITE_ENABLE_CONCURRENT +# define ISCONCURRENT (pBt->pMap!=0) #else -# define ISUNLOCKED 0 +# define ISCONCURRENT 0 #endif -#define REQUIRE_PTRMAP (ISAUTOVACUUM || ISUNLOCKED) +#define REQUIRE_PTRMAP (ISAUTOVACUUM || ISCONCURRENT) /* ** This structure is passed around through all the sanity checking routines diff --git a/src/build.c b/src/build.c index ed216cb292..d37ddde269 100644 --- a/src/build.c +++ b/src/build.c @@ -3862,7 +3862,7 @@ void sqlite3BeginTransaction(Parse *pParse, int type){ sqlite3VdbeUsesBtree(v, i); } } - sqlite3VdbeAddOp3(v, OP_AutoCommit, 0, 0, (type==TK_UNLOCKED)); + sqlite3VdbeAddOp3(v, OP_AutoCommit, 0, 0, (type==TK_CONCURRENT)); } /* diff --git a/src/pager.c b/src/pager.c index 9942211edb..ecdad403dc 100644 --- a/src/pager.c +++ b/src/pager.c @@ -657,8 +657,8 @@ struct Pager { u32 cksumInit; /* Quasi-random value added to every checksum */ u32 nSubRec; /* Number of records written to sub-journal */ Bitvec *pInJournal; /* One bit for each page in the database file */ -#ifdef SQLITE_ENABLE_UNLOCKED - Bitvec *pAllRead; /* Pages read within current UNLOCKED trans. */ +#ifdef SQLITE_ENABLE_CONCURRENT + Bitvec *pAllRead; /* Pages read within current CONCURRENT trans. */ #endif sqlite3_file *fd; /* File descriptor for database */ sqlite3_file *jfd; /* File descriptor for main journal */ @@ -1747,7 +1747,7 @@ static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ static void pagerFreeBitvecs(Pager *pPager){ sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; -#ifdef SQLITE_ENABLE_UNLOCKED +#ifdef SQLITE_ENABLE_CONCURRENT sqlite3BitvecDestroy(pPager->pAllRead); pPager->pAllRead = 0; #endif @@ -3035,10 +3035,10 @@ static int pagerRollbackWal(Pager *pPager){ rc = sqlite3WalUndo(pPager->pWal, pagerUndoCallback, (void *)pPager); pList = sqlite3PcacheDirtyList(pPager->pPCache); - /* If this is an UNLOCKED transaction, then page 1 must be reread from + /* If this is an CONCURRENT transaction, then page 1 must be reread from ** the db file, even if it is not dirty. This is because the b-tree layer ** may have already zeroed the nFree and iTrunk header fields. */ -#ifdef SQLITE_ENABLE_UNLOCKED +#ifdef SQLITE_ENABLE_CONCURRENT if( rc==SQLITE_OK && (pList==0 || pList->pgno!=1) && pPager->pAllRead ){ rc = pagerUndoCallback((void*)pPager, 1); } @@ -4452,9 +4452,9 @@ static int pagerStress(void *p, PgHdr *pPg){ pPg->pDirty = 0; if( pagerUseWal(pPager) ){ - /* If the transaction is a "BEGIN UNLOCKED" transaction, the page + /* If the transaction is a "BEGIN CONCURRENT" transaction, the page ** cannot be flushed to disk. Return early in this case. */ -#ifdef SQLITE_ENABLE_UNLOCKED +#ifdef SQLITE_ENABLE_CONCURRENT if( pPager->pAllRead ) return SQLITE_OK; #endif @@ -5298,10 +5298,10 @@ int sqlite3PagerAcquire( } pPager->hasBeenUsed = 1; - /* If this is an UNLOCKED transaction and the page being read was + /* If this is an CONCURRENT transaction and the page being read was ** present in the database file when the transaction was opened, ** mark it as read in the pAllRead vector. */ -#ifdef SQLITE_ENABLE_UNLOCKED +#ifdef SQLITE_ENABLE_CONCURRENT if( pPager->pAllRead && pgno<=pPager->dbOrigSize ){ rc = sqlite3BitvecSet(pPager->pAllRead, pgno); if( rc!=SQLITE_OK ) goto pager_acquire_err; @@ -5593,7 +5593,7 @@ static int pager_open_journal(Pager *pPager){ ** functions need be called. ** ** If (exFlag<0) and the database is in WAL mode, do not take any locks. -** The transaction will run in UNLOCKED mode instead. +** The transaction will run in CONCURRENT mode instead. ** ** If the subjInMemory argument is non-zero, then any sub-journal opened ** within this transaction will be opened as an in-memory file. This @@ -5612,7 +5612,7 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ if( ALWAYS(pPager->eState==PAGER_READER) ){ assert( pPager->pInJournal==0 ); -#ifdef SQLITE_ENABLE_UNLOCKED +#ifdef SQLITE_ENABLE_CONCURRENT assert( pPager->pAllRead==0 ); #endif @@ -5633,7 +5633,7 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ ** The busy-handler is not invoked if another connection already ** holds the write-lock. If possible, the upper layer will call it. */ -#ifdef SQLITE_ENABLE_UNLOCKED +#ifdef SQLITE_ENABLE_CONCURRENT if( exFlag<0 ){ pPager->pAllRead = sqlite3BitvecCreate(pPager->dbSize); if( pPager->pAllRead==0 ){ @@ -5949,7 +5949,7 @@ int sqlite3PagerWrite(PgHdr *pPg){ ** to sqlite3PagerWrite(). In other words, return TRUE if it is ok ** to change the content of the page. */ -#if defined(SQLITE_ENABLE_UNLOCKED) || !defined(NDEBUG) +#if defined(SQLITE_ENABLE_CONCURRENT) || !defined(NDEBUG) int sqlite3PagerIswriteable(DbPage *pPg){ return pPg->flags & PGHDR_WRITEABLE; } @@ -6105,10 +6105,10 @@ int sqlite3PagerSync(Pager *pPager, const char *zMaster){ ** current write-transaction to the database file are held. If the db is ** in rollback mode, this means the EXCLUSIVE lock on the database file. ** -** Or, if this is a non-UNLOCKED transaction on a wal-mode database, this +** Or, if this is a non-CONCURRENT transaction on a wal-mode database, this ** function is a no-op. ** -** If this is an UNLOCKED transaction on a wal-mode database, this function +** If this is an CONCURRENT transaction on a wal-mode database, this function ** attempts to obtain the WRITER lock on the wal file and also checks to ** see that the transaction can be safely committed (does not commit with ** any other transaction committed since it was opened). @@ -6116,7 +6116,7 @@ int sqlite3PagerSync(Pager *pPager, const char *zMaster){ ** If the required locks are already held or successfully obtained and ** the transaction can be committed, SQLITE_OK is returned. If a required lock ** cannot be obtained, SQLITE_BUSY is returned. Or, if the current transaction -** is UNLOCKED and cannot be committed due to a conflict, SQLITE_BUSY_SNAPSHOT +** is CONCURRENT and cannot be committed due to a conflict, SQLITE_BUSY_SNAPSHOT ** is returned. Otherwise, if some other error occurs (IO error, OOM etc.), ** and SQLite error code is returned. */ @@ -6130,10 +6130,10 @@ int sqlite3PagerExclusiveLock(Pager *pPager, PgHdr *pPage1){ if( 0==pagerUseWal(pPager) ){ rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); } -#ifdef SQLITE_ENABLE_UNLOCKED +#ifdef SQLITE_ENABLE_CONCURRENT else{ if( pPager->pAllRead ){ - /* This is an UNLOCKED transaction. Attempt to lock the wal database + /* This is an CONCURRENT transaction. Attempt to lock the wal database ** here. If SQLITE_BUSY (but not SQLITE_BUSY_SNAPSHOT) is returned, ** invoke the busy-handler and try again for as long as it returns ** non-zero. */ @@ -6148,9 +6148,9 @@ int sqlite3PagerExclusiveLock(Pager *pPager, PgHdr *pPage1){ return rc; } -#ifdef SQLITE_ENABLE_UNLOCKED +#ifdef SQLITE_ENABLE_CONCURRENT /* -** This function is called as part of committing an UNLOCKED transaction. +** This function is called as part of committing an CONCURRENT transaction. ** At this point the wal WRITER lock is held, and all pages in the cache ** except for page 1 are compatible with the snapshot at the head of the ** wal file. @@ -6185,9 +6185,9 @@ void sqlite3PagerSetDbsize(Pager *pPager, Pgno nSz){ } /* -** Return true if this pager is currently within an UNLOCKED transaction. +** Return true if this pager is currently within an CONCURRENT transaction. */ -int sqlite3PagerIsUnlocked(Pager *pPager){ +int sqlite3PagerIsConcurrent(Pager *pPager){ return pPager->pAllRead!=0; } @@ -6200,7 +6200,7 @@ void sqlite3PagerDropExclusiveLock(Pager *pPager){ sqlite3WalEndWriteTransaction(pPager->pWal); } } -#endif /* ifdef SQLITE_ENABLE_UNLOCKED */ +#endif /* ifdef SQLITE_ENABLE_CONCURRENT */ /* diff --git a/src/pager.h b/src/pager.h index c097c97c5b..c9ebaf1116 100644 --- a/src/pager.h +++ b/src/pager.h @@ -195,7 +195,7 @@ void sqlite3PagerTruncateImage(Pager*,Pgno); void sqlite3PagerRekey(DbPage*, Pgno, u16); void sqlite3PagerDropExclusiveLock(Pager*); -int sqlite3PagerIsUnlocked(Pager*); +int sqlite3PagerIsConcurrent(Pager*); int sqlite3PagerIswriteable(DbPage*); int sqlite3PagerUpgradeSnapshot(Pager *pPager, DbPage*); void sqlite3PagerSetDbsize(Pager *pPager, Pgno); diff --git a/src/parse.y b/src/parse.y index 2dd58bdf55..1fc72f753d 100644 --- a/src/parse.y +++ b/src/parse.y @@ -121,7 +121,7 @@ transtype(A) ::= . {A = TK_DEFERRED;} transtype(A) ::= DEFERRED(X). {A = @X;} transtype(A) ::= IMMEDIATE(X). {A = @X;} transtype(A) ::= EXCLUSIVE(X). {A = @X;} -transtype(A) ::= UNLOCKED(X). {A = @X;} +transtype(A) ::= CONCURRENT(X). {A = @X;} cmd ::= COMMIT trans_opt. {sqlite3CommitTransaction(pParse);} cmd ::= END trans_opt. {sqlite3CommitTransaction(pParse);} cmd ::= ROLLBACK trans_opt. {sqlite3RollbackTransaction(pParse);} diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 2d295b01fa..39931f0870 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1150,7 +1150,7 @@ struct sqlite3 { u16 dbOptFlags; /* Flags to enable/disable optimizations */ u8 enc; /* Text encoding */ u8 autoCommit; /* The auto-commit flag. */ - u8 bUnlocked; /* Current transaction is "UNLOCKED" */ + u8 bConcurrent; /* Current transaction is "CONCURRENT" */ u8 temp_store; /* 1: file 2: memory 0: default */ u8 mallocFailed; /* True if we have seen a malloc failure */ u8 dfltLockMode; /* Default locking-mode for attached dbs */ diff --git a/src/test_config.c b/src/test_config.c index 4d2dece83a..d153697715 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -573,10 +573,10 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY); Tcl_SetVar2(interp, "sqlite_options", "truncate_opt", "1", TCL_GLOBAL_ONLY); #endif -#ifdef SQLITE_ENABLE_UNLOCKED - Tcl_SetVar2(interp, "sqlite_options", "unlocked", "1", TCL_GLOBAL_ONLY); +#ifdef SQLITE_ENABLE_CONCURRENT + Tcl_SetVar2(interp, "sqlite_options", "concurrent", "1", TCL_GLOBAL_ONLY); #else - Tcl_SetVar2(interp, "sqlite_options", "unlocked", "0", TCL_GLOBAL_ONLY); + Tcl_SetVar2(interp, "sqlite_options", "concurrent", "0", TCL_GLOBAL_ONLY); #endif #ifdef SQLITE_OMIT_UTF16 diff --git a/src/vacuum.c b/src/vacuum.c index 46f2249ebb..eafaa66165 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -356,7 +356,7 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ ** is closed by the DETACH. */ db->autoCommit = 1; - assert( db->bUnlocked==0 ); + assert( db->bConcurrent==0 ); if( pDb ){ sqlite3BtreeClose(pDb->pBt); diff --git a/src/vdbe.c b/src/vdbe.c index 3cdb252fdf..67703a781a 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2895,7 +2895,7 @@ case OP_Savepoint: { ** is committed. */ int isTransaction = pSavepoint->pNext==0 && db->isTransactionSavepoint; - assert( db->bUnlocked==0 || db->isTransactionSavepoint==0 ); + assert( db->bConcurrent==0 || db->isTransactionSavepoint==0 ); if( isTransaction && p1==SAVEPOINT_RELEASE ){ if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ goto vdbe_return; @@ -2979,7 +2979,7 @@ case OP_Savepoint: { ** there are active writing VMs or active VMs that use shared cache. ** ** If P3 is non-zero, then this instruction is being executed as part of -** a "BEGIN UNLOCKED" command. +** a "BEGIN CONCURRENT" command. ** ** This instruction causes the VM to halt. */ @@ -2987,25 +2987,25 @@ case OP_AutoCommit: { int desiredAutoCommit; int iRollback; int turnOnAC; - int bUnlocked; + int bConcurrent; int hrc; desiredAutoCommit = pOp->p1; iRollback = pOp->p2; - bUnlocked = pOp->p3; + bConcurrent = pOp->p3; turnOnAC = desiredAutoCommit && !db->autoCommit; assert( desiredAutoCommit==1 || desiredAutoCommit==0 ); assert( desiredAutoCommit==1 || iRollback==0 ); - assert( desiredAutoCommit==0 || bUnlocked==0 ); - assert( db->autoCommit==0 || db->bUnlocked==0 ); + assert( desiredAutoCommit==0 || bConcurrent==0 ); + assert( db->autoCommit==0 || db->bConcurrent==0 ); assert( db->nVdbeActive>0 ); /* At least this one VM is active */ assert( p->bIsReader ); if( turnOnAC && !iRollback - && (db->nVdbeWrite>0 || (db->bUnlocked && db->nVdbeActive>1)) + && (db->nVdbeWrite>0 || (db->bConcurrent && db->nVdbeActive>1)) ){ /* A transaction may only be committed if there are no other active - ** writer VMs. If the transaction is UNLOCKED, then it may only be + ** writer VMs. If the transaction is CONCURRENT, then it may only be ** committed if there are no active VMs at all (readers or writers). ** ** If this instruction is a COMMIT and the transaction may not be @@ -3020,7 +3020,7 @@ case OP_AutoCommit: { assert( desiredAutoCommit==1 ); sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); db->autoCommit = 1; - db->bUnlocked = 0; + db->bConcurrent = 0; }else if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ goto vdbe_return; }else{ @@ -3034,7 +3034,7 @@ case OP_AutoCommit: { rc = SQLITE_BUSY; goto vdbe_return; } - db->bUnlocked = (u8)bUnlocked; + db->bConcurrent = (u8)bConcurrent; assert( db->nStatement==0 ); sqlite3CloseSavepoints(db); if( p->rc==SQLITE_OK ){ @@ -3227,12 +3227,12 @@ case OP_SetCookie: { /* in3 */ assert( sqlite3SchemaMutexHeld(db, pOp->p1, 0) ); pIn3 = &aMem[pOp->p3]; sqlite3VdbeMemIntegerify(pIn3); -#ifdef SQLITE_ENABLE_UNLOCKED - if( db->bUnlocked +#ifdef SQLITE_ENABLE_CONCURRENT + if( db->bConcurrent && (pOp->p2==BTREE_USER_VERSION || pOp->p2==BTREE_APPLICATION_ID) ){ rc = SQLITE_ERROR; - sqlite3VdbeError(p, "cannot modify %s within UNLOCKED transaction", + sqlite3VdbeError(p, "cannot modify %s within CONCURRENT transaction", pOp->p2==BTREE_USER_VERSION ? "user_version" : "application_id" ); break; @@ -3242,7 +3242,7 @@ case OP_SetCookie: { /* in3 */ rc = sqlite3BtreeUpdateMeta(pDb->pBt, pOp->p2, (int)pIn3->u.i); if( pOp->p2==BTREE_SCHEMA_VERSION ){ /* When the schema cookie changes, record the new cookie internally */ - assert( db->bUnlocked==0 ); + assert( db->bConcurrent==0 ); pDb->pSchema->schema_cookie = (int)pIn3->u.i; db->flags |= SQLITE_InternChanges; }else if( pOp->p2==BTREE_FILE_FORMAT ){ @@ -6115,11 +6115,11 @@ case OP_Expire: { */ case OP_TableLock: { u8 isWriteLock = (u8)pOp->p3; -#ifdef SQLITE_ENABLE_UNLOCKED - if( isWriteLock && db->bUnlocked && pOp->p2==1 ){ +#ifdef SQLITE_ENABLE_CONCURRENT + if( isWriteLock && db->bConcurrent && pOp->p2==1 ){ rc = SQLITE_ERROR; sqlite3VdbeError(p, - "cannot modify database schema within UNLOCKED transaction"); + "cannot modify database schema within CONCURRENT transaction"); rc = SQLITE_ERROR; break; } diff --git a/src/vdbeaux.c b/src/vdbeaux.c index ea467e08a3..4aef2ee08e 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2025,8 +2025,8 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ } } -#ifdef SQLITE_ENABLE_UNLOCKED - if( db->bUnlocked && (rc & 0xFF)==SQLITE_BUSY ){ +#ifdef SQLITE_ENABLE_CONCURRENT + if( db->bConcurrent && (rc & 0xFF)==SQLITE_BUSY ){ /* An SQLITE_BUSY or SQLITE_BUSY_SNAPSHOT was encountered while ** attempting to take the WRITER lock on a wal file. Release the ** WRITER locks on all wal files and return early. */ @@ -2443,7 +2443,7 @@ int sqlite3VdbeHalt(Vdbe *p){ sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; - db->bUnlocked = 0; + db->bConcurrent = 0; p->nChange = 0; } } @@ -2506,7 +2506,7 @@ int sqlite3VdbeHalt(Vdbe *p){ sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; - db->bUnlocked = 0; + db->bConcurrent = 0; p->nChange = 0; } } @@ -2528,7 +2528,7 @@ int sqlite3VdbeHalt(Vdbe *p){ sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; - db->bUnlocked = 0; + db->bConcurrent = 0; p->nChange = 0; } } diff --git a/src/wal.c b/src/wal.c index 9a2152424a..4db4209750 100644 --- a/src/wal.c +++ b/src/wal.c @@ -768,7 +768,7 @@ static const char *walLockName(int lockIdx){ /* ** Set or release locks on the WAL. Locks are either shared or exclusive. ** A lock cannot be moved directly between shared and exclusive - it must go -** through the unlocked state first. +** through the concurrent state first. ** ** In locking_mode=EXCLUSIVE, all of these routines become no-ops. */ @@ -1068,7 +1068,7 @@ static int walIndexRecover(Wal *pWal){ /* Obtain an exclusive lock on all byte in the locking range not already ** locked by the caller. The caller is guaranteed to have locked the ** WAL_WRITE_LOCK byte, and may have also locked the WAL_CKPT_LOCK byte. - ** If successful, the same bytes that are locked here are unlocked before + ** If successful, the same bytes that are locked here are concurrent before ** this function returns. */ assert( pWal->ckptLock==1 || pWal->ckptLock==0 ); @@ -2608,14 +2608,14 @@ static int walUpgradeReadlock(Wal *pWal){ } -#ifdef SQLITE_ENABLE_UNLOCKED +#ifdef SQLITE_ENABLE_CONCURRENT /* -** This function is only ever called when committing a "BEGIN UNLOCKED" +** This function is only ever called when committing a "BEGIN CONCURRENT" ** transaction. It may be assumed that no frames have been written to ** the wal file. The second parameter is a pointer to the in-memory ** representation of page 1 of the database (which may or may not be ** dirty). The third is a bitvec with a bit set for each page in the -** database file that was read by the current unlocked transaction. +** database file that was read by the current concurrent transaction. ** ** This function performs three tasks: ** @@ -2626,7 +2626,7 @@ static int walUpgradeReadlock(Wal *pWal){ ** it was opened, and ** ** 3) It ejects any non-dirty pages from the page-cache that have been -** written by another client since the UNLOCKED transaction was started +** written by another client since the CONCURRENT transaction was started ** (so as to avoid ending up with an inconsistent cache after the ** current transaction is committed). ** @@ -2657,7 +2657,7 @@ int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPage1, Bitvec *pAllRead){ if( walIndexLoadHdr(pWal, &head) ){ /* This branch is taken if the wal-index header is corrupted. This ** occurs if some other writer has crashed while committing a - ** transaction to this database since the current unlocked transaction + ** transaction to this database since the current concurrent transaction ** was opened. */ rc = SQLITE_BUSY_SNAPSHOT; }else if( memcmp(&pWal->hdr, (void*)&head, sizeof(WalIndexHdr))!=0 ){ @@ -2698,13 +2698,13 @@ int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPage1, Bitvec *pAllRead){ } }else if( sqlite3BitvecTestNotNull(pAllRead, aPgno[i]) ){ sqlite3_log(SQLITE_OK, - "cannot commit UNLOCKED transaction (conflict at page %d)", + "cannot commit CONCURRENT transaction (conflict at page %d)", (int)aPgno[i] ); rc = SQLITE_BUSY_SNAPSHOT; }else if( (pPg = sqlite3PagerLookup(pPager, aPgno[i])) ){ /* Page aPgno[i], which is present in the pager cache, has been - ** modified since the current UNLOCKED transaction was started. + ** modified since the current CONCURRENT transaction was started. ** However it was not read by the current transaction, so is not ** a conflict. There are two possibilities: (a) the page was ** allocated at the of the file by the current transaction or @@ -2735,14 +2735,14 @@ int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPage1, Bitvec *pAllRead){ } /* -** This function is called as part of committing an UNLOCKED transaction. +** This function is called as part of committing an CONCURRENT transaction. ** It is assumed that sqlite3WalLockForCommit() has already been successfully ** called and so (a) the WRITER lock is held and (b) it is known that the ** wal-index-header stored in shared memory is not corrupt. ** ** Before returning, this function upgrades the client so that it is ** operating on the database snapshot currently at the head of the wal file -** (even if the UNLOCKED transaction ran against an older snapshot). +** (even if the CONCURRENT transaction ran against an older snapshot). ** ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ @@ -2760,7 +2760,7 @@ int sqlite3WalUpgradeSnapshot(Wal *pWal){ } return rc; } -#endif /* SQLITE_ENABLE_UNLOCKED */ +#endif /* SQLITE_ENABLE_CONCURRENT */ /* ** End a write transaction. The commit has already been done. This diff --git a/test/unlocked.test b/test/concurrent.test similarity index 85% rename from test/unlocked.test rename to test/concurrent.test index 0e22b97fa1..4f542ee797 100644 --- a/test/unlocked.test +++ b/test/concurrent.test @@ -13,9 +13,9 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/lock_common.tcl -set ::testprefix unlocked +set ::testprefix concurrent -ifcapable !unlocked { +ifcapable !concurrent { finish_test return } @@ -26,7 +26,7 @@ do_execsql_test 1.0 { do_execsql_test 1.1 { CREATE TABLE t1(k INTEGER PRIMARY KEY, v); - BEGIN UNLOCKED; + BEGIN CONCURRENT; INSERT INTO t1 VALUES(1, 'abcd'); COMMIT; } @@ -36,7 +36,7 @@ do_execsql_test 1.2 { } {1 abcd} do_execsql_test 1.3 { - BEGIN UNLOCKED; + BEGIN CONCURRENT; INSERT INTO t1 VALUES(2, 'efgh'); ROLLBACK; } @@ -47,10 +47,10 @@ do_execsql_test 1.4 { #------------------------------------------------------------------------- -# UNLOCKED transactions cannot do cache spills. +# CONCURRENT transactions cannot do cache spills. # foreach {tn trans spill} { - 1 {BEGIN UNLOCKED} 0 + 1 {BEGIN CONCURRENT} 0 2 {BEGIN} 1 } { do_test 1.5.$tn { @@ -72,7 +72,7 @@ foreach {tn trans spill} { } #------------------------------------------------------------------------- -# UNLOCKED transactions man not be committed while there are active +# CONCURRENT transactions man not be committed while there are active # readers. do_execsql_test 1.6.setup { DROP TABLE t1; @@ -82,7 +82,7 @@ do_execsql_test 1.6.setup { INSERT INTO t1 VALUES(5, 6); } foreach {tn trans commit_ok} { - 1 {BEGIN UNLOCKED} 0 + 1 {BEGIN CONCURRENT} 0 2 {BEGIN} 1 } { do_test 1.6.$tn.1 { @@ -105,7 +105,7 @@ foreach {tn trans commit_ok} { } #------------------------------------------------------------------------- -# UNLOCKED transactions may not modify the db schema. +# CONCURRENT transactions may not modify the db schema. # foreach {tn sql} { 1 { CREATE TABLE xx(a, b) } @@ -115,9 +115,9 @@ foreach {tn sql} { 5 { CREATE TEMP TABLE xx(a, b) } } { do_catchsql_test 1.7.$tn.1 " - BEGIN UNLOCKED; + BEGIN CONCURRENT; $sql - " {1 {cannot modify database schema within UNLOCKED transaction}} + " {1 {cannot modify database schema within CONCURRENT transaction}} do_execsql_test 1.7.$tn.2 { SELECT sql FROM sqlite_master; @@ -128,8 +128,8 @@ foreach {tn sql} { } #------------------------------------------------------------------------- -# If an auto-vacuum database is written within an UNLOCKED transaction, it -# is handled in the same way as for a non-UNLOCKED transaction. +# If an auto-vacuum database is written within an CONCURRENT transaction, it +# is handled in the same way as for a non-CONCURRENT transaction. # reset_db do_execsql_test 1.8.1 { @@ -140,20 +140,20 @@ do_execsql_test 1.8.1 { } {wal} do_execsql_test 1.8.2 { - BEGIN UNLOCKED; + BEGIN CONCURRENT; SELECT * FROM t1; COMMIT; } {x y} do_catchsql_test 1.8.3 { - BEGIN UNLOCKED; + BEGIN CONCURRENT; INSERT INTO t1 VALUES('a', 'b'); } {0 {}} do_test 1.8.4 { sqlite3 db2 test.db catchsql { - BEGIN UNLOCKED; + BEGIN CONCURRENT; INSERT INTO t1 VALUES('c', 'd'); } db2 } {1 {database is locked}} @@ -166,10 +166,10 @@ do_test 1.8.5 { do_multiclient_test tn { #----------------------------------------------------------------------- - # 1. Start an UNLOCKED transaction using [db1]. + # 1. Start an CONCURRENT transaction using [db1]. # # 2. Start and then rollback a regular transaction using [db2]. This - # can be done as the ongoing [db1] transaction is UNLOCKED. + # can be done as the ongoing [db1] transaction is CONCURRENT. # # 3. The [db1] transaction can now be committed, as [db2] has relinquished # the write lock. @@ -181,7 +181,7 @@ do_multiclient_test tn { INSERT INTO t1 VALUES(1, 'one'); } sql1 { - BEGIN UNLOCKED; + BEGIN CONCURRENT; INSERT INTO t1 VALUES(2, 'two'); } code1 { sqlite3_get_autocommit db } @@ -201,7 +201,7 @@ do_multiclient_test tn { } {1 one 2 two} #----------------------------------------------------------------------- - # 1. Start an UNLOCKED transaction using [db1]. + # 1. Start an CONCURRENT transaction using [db1]. # # 2. Commit a transaction using [db2]. # @@ -210,7 +210,7 @@ do_multiclient_test tn { # do_test 2.$tn.2.1 { sql1 { - BEGIN UNLOCKED; + BEGIN CONCURRENT; INSERT INTO t1 VALUES(-1, 'hello world'); } } {} @@ -238,7 +238,7 @@ do_multiclient_test tn { } {-1 {hello world} 1 one 2 two} #----------------------------------------------------------------------- - # 1. Start an UNLOCKED transaction using [db1]. + # 1. Start an CONCURRENT transaction using [db1]. # # 2. Open a transaction using [db2]. # @@ -250,7 +250,7 @@ do_multiclient_test tn { # do_test 2.$tn.3.1 { sql1 { - BEGIN UNLOCKED; + BEGIN CONCURRENT; INSERT INTO t1 VALUES(4, 'four'); } } {} @@ -293,12 +293,12 @@ do_multiclient_test tn { } {} do_test 2.$tn.4.2 { sql2 { - BEGIN UNLOCKED; + BEGIN CONCURRENT; INSERT INTO t2 VALUES('i', 'n'); } sql1 { - BEGIN UNLOCKED; + BEGIN CONCURRENT; INSERT INTO t1 VALUES(5, 'five'); COMMIT; } @@ -319,16 +319,16 @@ do_multiclient_test tn { #----------------------------------------------------------------------- # The "schema cookie" issue. # - # 1. Begin and UNLOCKED write to "t1" using [db] + # 1. Begin and CONCURRENT write to "t1" using [db] # # 2. Create an index on t1 using [db2]. # - # 3. Attempt to commit the UNLOCKED write. This is an SQLITE_BUSY_SNAPSHOT, + # 3. Attempt to commit the CONCURRENT write. This is an SQLITE_BUSY_SNAPSHOT, # even though there is no page collision. # do_test 2.$tn.5.1 { sql1 { - BEGIN UNLOCKED; + BEGIN CONCURRENT; INSERT INTO t1 VALUES(6, 'six'); } } {} @@ -348,15 +348,15 @@ do_multiclient_test tn { #----------------------------------------------------------------------- # - # 1. Begin an UNLOCKED write to "t1" using [db] + # 1. Begin an CONCURRENT write to "t1" using [db] # # 2. Lots of inserts into t2. Enough to grow the db file and modify page 1. # - # 3. Check that the UNLOCKED transaction can not be committed. + # 3. Check that the CONCURRENT transaction can not be committed. # do_test 2.$tn.6.1 { sql1 { - BEGIN UNLOCKED; + BEGIN CONCURRENT; INSERT INTO t1 VALUES(6, 'six'); } } {} @@ -384,17 +384,17 @@ do_multiclient_test tn { #----------------------------------------------------------------------- # - # 1. Begin an big UNLOCKED write to "t1" using [db] - large enough to + # 1. Begin an big CONCURRENT write to "t1" using [db] - large enough to # grow the db file. # # 2. Lots of inserts into t2. Also enough to grow the db file. # - # 3. Check that the UNLOCKED transaction cannot be committed (due to a clash + # 3. Check that the CONCURRENT transaction cannot be committed (due to a clash # on page 1 - the db size field). # do_test 2.$tn.7.1 { sql1 { - BEGIN UNLOCKED; + BEGIN CONCURRENT; WITH src(a,b) AS ( VALUES(10000,10000) UNION ALL SELECT a+1,b+1 FROM src WHERE a<20000 ) INSERT INTO t1 SELECT * FROM src; @@ -427,20 +427,20 @@ do_execsql_test 3.0 { PRAGMA user_version = 10; } {wal} do_execsql_test 3.1 { - BEGIN UNLOCKED; + BEGIN CONCURRENT; INSERT INTO t1 VALUES('c', 'd'); SELECT * FROM t1; } {a b c d} do_catchsql_test 3.2 { PRAGMA user_version = 11; -} {1 {cannot modify user_version within UNLOCKED transaction}} +} {1 {cannot modify user_version within CONCURRENT transaction}} do_execsql_test 3.3 { PRAGMA user_version; SELECT * FROM t1; } {10 a b c d} do_catchsql_test 3.4 { PRAGMA application_id = 11; -} {1 {cannot modify application_id within UNLOCKED transaction}} +} {1 {cannot modify application_id within CONCURRENT transaction}} do_execsql_test 3.5 { COMMIT; PRAGMA user_version; diff --git a/test/unlocked2.test b/test/concurrent2.test similarity index 87% rename from test/unlocked2.test rename to test/concurrent2.test index f28d62e0a1..d058733e48 100644 --- a/test/unlocked2.test +++ b/test/concurrent2.test @@ -13,9 +13,9 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/lock_common.tcl -set ::testprefix unlocked2 +set ::testprefix concurrent2 -ifcapable !unlocked { +ifcapable !concurrent { finish_test return } @@ -31,11 +31,11 @@ do_multiclient_test tn { } {wal} do_test 1.$tn.5 { sql3 { PRAGMA integrity_check } } {ok} - # Test that an UNLOCKED transaction that allocates/frees no pages does + # Test that an CONCURRENT transaction that allocates/frees no pages does # not conflict with a transaction that does allocate pages. do_test 1.$tn.2 { sql1 { - BEGIN UNLOCKED; + BEGIN CONCURRENT; INSERT INTO t1 VALUES(4); } sql2 { @@ -47,11 +47,11 @@ do_multiclient_test tn { } {} do_test 1.$tn.5 { sql3 { PRAGMA integrity_check } } {ok} - # But that an UNLOCKED transaction does conflict with a transaction + # But that an CONCURRENT transaction does conflict with a transaction # that modifies the db schema. do_test 1.$tn.3 { sql1 { - BEGIN UNLOCKED; + BEGIN CONCURRENT; INSERT INTO t1 VALUES(5); } sql2 { @@ -61,12 +61,12 @@ do_multiclient_test tn { } {1 {database is locked}} do_test 1.$tn.5 { sql3 { PRAGMA integrity_check } } {ok} - # Test that an UNLOCKED transaction that allocates at least one page + # Test that an CONCURRENT transaction that allocates at least one page # does not conflict with a transaction that allocates no pages. do_test 1.$tn.4 { sql1 { ROLLBACK; - BEGIN UNLOCKED; + BEGIN CONCURRENT; INSERT INTO t1 VALUES(randomblob(1500)); } sql2 { @@ -91,7 +91,7 @@ do_multiclient_test tn { do_test 2.$tn.2 { sql1 { - BEGIN UNLOCKED; + BEGIN CONCURRENT; INSERT INTO t1 VALUES(randomblob(1500)); } sql2 { @@ -104,7 +104,7 @@ do_multiclient_test tn { do_test 2.$tn.4 { sql1 { - BEGIN UNLOCKED; + BEGIN CONCURRENT; DELETE FROM t1; } sql2 { @@ -124,7 +124,7 @@ do_multiclient_test tn { } sql1 { - BEGIN UNLOCKED; + BEGIN CONCURRENT; DELETE FROM t1 WHERE rowid=2; } @@ -139,15 +139,15 @@ do_multiclient_test tn { } #------------------------------------------------------------------------- -# When an UNLOCKED transaction is opened on a database, the nFree and +# When an CONCURRENT transaction is opened on a database, the nFree and # iTrunk header fields of the cached version of page 1 are both set -# to 0. This allows an UNLOCKED transaction to use its own private +# to 0. This allows an CONCURRENT transaction to use its own private # free-page-list, which is merged with the main database free-list when # the transaction is committed. # # The following tests check that nFree/iTrunk are correctly restored if -# an UNLOCKED transaction is rolled back, and that savepoint rollbacks -# that occur within UNLOCKED transactions do not incorrectly restore +# an CONCURRENT transaction is rolled back, and that savepoint rollbacks +# that occur within CONCURRENT transactions do not incorrectly restore # these fields to their on-disk values. # reset_db @@ -159,7 +159,7 @@ do_execsql_test 3.0 { } {wal} do_execsql_test 3.1 { - BEGIN UNLOCKED; + BEGIN CONCURRENT; INSERT INTO t1 VALUES(1, 2); ROLLBACK; } @@ -168,7 +168,7 @@ do_execsql_test 3.2 { PRAGMA integrity_check } {ok} do_execsql_test 3.3 { PRAGMA freelist_count } {2} do_execsql_test 3.4.1 { - BEGIN UNLOCKED; + BEGIN CONCURRENT; PRAGMA freelist_count; } {2} do_execsql_test 3.4.2 { @@ -184,7 +184,7 @@ do_execsql_test 3.4.5 { COMMIT; PRAGMA freelist_count } {2} do_execsql_test 3.4.6 { PRAGMA integrity_check } {ok} do_execsql_test 3.5.1 { - BEGIN UNLOCKED; + BEGIN CONCURRENT; UPDATE t1 SET x=randomblob(10) WHERE y=555; PRAGMA freelist_count; } {0} @@ -195,7 +195,7 @@ do_execsql_test 3.5.2 { do_execsql_test 3.5.3 { PRAGMA integrity_check } {ok} #------------------------------------------------------------------------- -# Test that nothing goes wrong if an UNLOCKED transaction allocates a +# Test that nothing goes wrong if an CONCURRENT transaction allocates a # page at the end of the file, frees it within the same transaction, and # then has to move the same page to avoid a conflict on COMMIT. # @@ -210,7 +210,7 @@ do_multiclient_test tn { do_test 4.$tn.2 { sql1 { - BEGIN UNLOCKED; + BEGIN CONCURRENT; INSERT INTO t1 VALUES(randomblob(1500)); INSERT INTO t1 VALUES(randomblob(1500)); DELETE FROM t1 WHERE rowid = 1; @@ -243,7 +243,7 @@ do_multiclient_test tn { do_test 5.$tn.2 { sql1 { - BEGIN UNLOCKED; + BEGIN CONCURRENT; INSERT INTO t2 VALUES(randomblob(1500)); PRAGMA page_count; } @@ -275,7 +275,7 @@ do_multiclient_test tn { do_test 6.$tn.2 { sql1 { - BEGIN UNLOCKED; + BEGIN CONCURRENT; INSERT INTO t1 VALUES(randomblob(1500)); INSERT INTO t1 VALUES(randomblob(1500)); } diff --git a/test/unlocked3.test b/test/concurrent3.test similarity index 93% rename from test/unlocked3.test rename to test/concurrent3.test index 36299db381..25e5d7c6c9 100644 --- a/test/unlocked3.test +++ b/test/concurrent3.test @@ -9,17 +9,17 @@ # #*********************************************************************** # -# Tests for transactions started with BEGIN UNLOCKED. The tests in this +# Tests for transactions started with BEGIN CONCURRENT. The tests in this # file focus on testing that deferred page allocation works properly. # set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/lock_common.tcl -set ::testprefix unlocked3 +set ::testprefix concurrent3 if {$AUTOVACUUM} { finish_test ; return } -ifcapable !unlocked { +ifcapable !concurrent { finish_test return } @@ -110,7 +110,7 @@ foreach {tn oplist} { foreach db $DBLIST { sqlite3 $db test.db } do_test 1.$tn { - foreach db $DBLIST { $db eval "BEGIN UNLOCKED" } + foreach db $DBLIST { $db eval "BEGIN CONCURRENT" } foreach op $oplist { set iTbl [string range $op 0 0] diff --git a/tool/mkkeywordhash.c b/tool/mkkeywordhash.c index d1810cbeb9..50585b1d09 100644 --- a/tool/mkkeywordhash.c +++ b/tool/mkkeywordhash.c @@ -171,6 +171,7 @@ static Keyword aKeywordTable[] = { { "COLLATE", "TK_COLLATE", ALWAYS }, { "COLUMN", "TK_COLUMNKW", ALTER }, { "COMMIT", "TK_COMMIT", ALWAYS }, + { "CONCURRENT", "TK_CONCURRENT", ALWAYS }, { "CONFLICT", "TK_CONFLICT", CONFLICT }, { "CONSTRAINT", "TK_CONSTRAINT", ALWAYS }, { "CREATE", "TK_CREATE", ALWAYS }, @@ -262,7 +263,6 @@ static Keyword aKeywordTable[] = { { "TRIGGER", "TK_TRIGGER", TRIGGER }, { "UNION", "TK_UNION", COMPOUND }, { "UNIQUE", "TK_UNIQUE", ALWAYS }, - { "UNLOCKED", "TK_UNLOCKED", ALWAYS }, { "UPDATE", "TK_UPDATE", ALWAYS }, { "USING", "TK_USING", ALWAYS }, { "VACUUM", "TK_VACUUM", VACUUM }, From 0cd4f69c0cbb5d0e1eb6582564f91cc10f2c91fd Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 24 Aug 2015 22:06:02 +0000 Subject: [PATCH 018/179] Remove duplicated line of code. FossilOrigin-Name: 47280f2a2b7cc83bf11ab86284204f565c278c55 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/vdbe.c | 1 - 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index ceae322d8a..ee49634063 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\s"BEGIN\sUNLOCKED"\sto\s"BEGIN\sCONCURRENT". -D 2015-08-24T19:56:04.567 +C Remove\sduplicated\sline\sof\scode. +D 2015-08-24T22:06:02.586 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -400,7 +400,7 @@ F src/update.c 487747b328b7216bb7f6af0695d6937d5c9e605f F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c bc9dd64b5db544218b871b66243871c202b2781f F src/vacuum.c 983cc3754718ef169a6ea9aef86798bd28106f21 -F src/vdbe.c 84ac0688d7e1921d8e802f2e4e1a38273e069d0c +F src/vdbe.c 791a28761cf6176d11ec3e41cdf8e749c65384ff F src/vdbe.h 7a75045d879118b9d3af7e8b3c108f2f27c51473 F src/vdbeInt.h 8b54e01ad0463590e7cffabce0bc36da9ee4f816 F src/vdbeapi.c bda74ef4b5103d7b4a4be36f936d3cf2b56a7d6f @@ -523,9 +523,9 @@ F test/collate9.test 3adcc799229545940df2f25308dd1ad65869145a F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b -F test/concurrent.test 631e398b127a13ddfbcc8e64f438d9f82b261ffe w test/unlocked.test -F test/concurrent2.test 21a15630192ba92287070a1a58a6e497d393c55d w test/unlocked2.test -F test/concurrent3.test 7dcf81372c06cbac58e7e630aebf7292945947bb w test/unlocked3.test +F test/concurrent.test 631e398b127a13ddfbcc8e64f438d9f82b261ffe +F test/concurrent2.test 21a15630192ba92287070a1a58a6e497d393c55d +F test/concurrent3.test 7dcf81372c06cbac58e7e630aebf7292945947bb F test/conflict.test 841bcf7cabbfca39c577eb8411ea8601843b46a8 F test/conflict2.test 0d3af4fb534fa1bd020c79960bb56e4d52655f09 F test/conflict3.test dec0634c0f31dec9a4b01c63063e939f0cd21b6b @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5b9f272113d21fd606903509d6f830fe60fac039 -R 7a2246f9f176c887e1e1da04647372c5 -U dan -Z f0bce293b4f554fd07229d405fb3ff97 +P ba1ab858e2997c88dd7eee6e6893a8616d85c665 +R 9abcd1f9df86b1795253bff8998d1e7d +U mistachkin +Z ced970d02bef7e6e65b8cf8ae80899c7 diff --git a/manifest.uuid b/manifest.uuid index e856c7e395..408e86fc5b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ba1ab858e2997c88dd7eee6e6893a8616d85c665 \ No newline at end of file +47280f2a2b7cc83bf11ab86284204f565c278c55 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 67703a781a..bf14bd1173 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -6120,7 +6120,6 @@ case OP_TableLock: { rc = SQLITE_ERROR; sqlite3VdbeError(p, "cannot modify database schema within CONCURRENT transaction"); - rc = SQLITE_ERROR; break; } #endif From ac0a42233adf135f0cd1ac66e4f8dbc17f7e500a Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 25 Aug 2015 14:37:39 +0000 Subject: [PATCH 019/179] Fix a segfault that could occur following an OOM condition in the concurrent transaction code. FossilOrigin-Name: 231b5880228cf01efe3981bc8be3150d79b422e5 --- manifest | 15 ++++---- manifest.uuid | 2 +- src/btree.c | 2 +- test/concfault.test | 86 ++++++++++++++++++++++++++++++++++++++++++++ test/concurrent.test | 1 + 5 files changed, 97 insertions(+), 9 deletions(-) create mode 100644 test/concfault.test diff --git a/manifest b/manifest index acbd257352..b28ae7786e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\swith\sthis\sbranch. -D 2015-08-25T11:16:02.604 +C Fix\sa\ssegfault\sthat\scould\soccur\sfollowing\san\sOOM\scondition\sin\sthe\sconcurrent\stransaction\scode. +D 2015-08-25T14:37:39.823 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -279,7 +279,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 4d9134dc988a87838c06056c89c0e8c4700a0452 F src/bitvec.c d1f21d7d91690747881f03940584f4cc548c9d3d F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 -F src/btree.c f4db20fab7bf5ebd9f12ef7e3a925ca3c2c34fa8 +F src/btree.c a76329691503fc68ec374791aaf4074dc8686b3a F src/btree.h 00d4cdb747c4172a5566faf037116985dbbc377e F src/btreeInt.h 171864bcd81635583dab7b8a04b19b454b18ef80 F src/build.c 1b5814e0eeaba096ae3ee17ebd139d2a25af7250 @@ -523,7 +523,8 @@ F test/collate9.test 3adcc799229545940df2f25308dd1ad65869145a F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b -F test/concurrent.test 631e398b127a13ddfbcc8e64f438d9f82b261ffe +F test/concfault.test 500f17c3fcfe7705114422bcc6ddd3c740001a43 +F test/concurrent.test 26c2d49abbf4847ceed9bf8cf7fbe9a2a4ffc70c F test/concurrent2.test 21a15630192ba92287070a1a58a6e497d393c55d F test/concurrent3.test 7dcf81372c06cbac58e7e630aebf7292945947bb F test/conflict.test 841bcf7cabbfca39c577eb8411ea8601843b46a8 @@ -1381,7 +1382,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 47280f2a2b7cc83bf11ab86284204f565c278c55 8f1d9f1f308518522ebb1eaebb9b184b4ac30924 -R 4996e84b0ad088ebe2e29da93ebd1db4 +P 3e7d6dd62dfa63d7def00bd00ac055a606a0c80d +R 7e8bf99bd9ee49e1baaabce6c0c30e59 U dan -Z 753c22f173f485729e70306b30e77d72 +Z 16262c3600df8d68cd85cb26ef84be69 diff --git a/manifest.uuid b/manifest.uuid index fa55886f4f..9e2c009bc9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3e7d6dd62dfa63d7def00bd00ac055a606a0c80d \ No newline at end of file +231b5880228cf01efe3981bc8be3150d79b422e5 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index e391255a41..9b90f58d07 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3904,9 +3904,9 @@ static int btreeRelocateRange( }else{ rc = allocateBtreePage(pBt, &pFree, &iNew, iFirst-1, BTALLOC_LE); assert( rc!=SQLITE_OK || iNeweType, pEntry->parent,iNew,1); releasePage(pPg); diff --git a/test/concfault.test b/test/concfault.test new file mode 100644 index 0000000000..6d409c8d74 --- /dev/null +++ b/test/concfault.test @@ -0,0 +1,86 @@ +# 2015 Aug 25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains fault injection tests designed to test the concurrent +# transactions feature. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set testprefix concfault + +# This test will not work with an in-memory journal, as the database will +# become corrupt if an error is injected into a transaction after it starts +# writing data out to the db file. +ifcapable !concurrent { + finish_test + return +} + +do_test 1-pre1 { + execsql { + PRAGMA journal_mode = wal; + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES(randomblob(1000), randomblob(100)); + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + DELETE FROM t1 WHERE rowid%2; + } + faultsim_save_and_close +} {} + +do_faultsim_test 1.1 -prep { + faultsim_restore_and_reopen +} -body { + execsql { + BEGIN CONCURRENT; + INSERT INTO t1 VALUES(randomblob(1000), randomblob(100)); + COMMIT; + } +} -test { + faultsim_test_result {0 {}} + catchsql { ROLLBACK } + faultsim_integrity_check +} + +do_faultsim_test 1.2 -prep { + faultsim_restore_and_reopen +} -body { + execsql { + BEGIN CONCURRENT; + INSERT INTO t1 VALUES(randomblob(1000), randomblob(100)); + ROLLBACK; + } +} -test { + faultsim_test_result {0 {}} + catchsql { ROLLBACK } + faultsim_integrity_check +} + +do_faultsim_test 1.3 -prep { + faultsim_restore_and_reopen +} -body { + execsql { + BEGIN CONCURRENT; + DELETE FROM t1; + COMMIT; + } +} -test { + faultsim_test_result {0 {}} + catchsql { ROLLBACK } + faultsim_integrity_check +} + +finish_test + diff --git a/test/concurrent.test b/test/concurrent.test index 4f542ee797..cf7d59257c 100644 --- a/test/concurrent.test +++ b/test/concurrent.test @@ -448,5 +448,6 @@ do_execsql_test 3.5 { SELECT * FROM t1; } {10 0 a b c d} + finish_test From 4073b26a20526e08ddadd3137d641cb8b6a627ae Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 25 Aug 2015 16:01:04 +0000 Subject: [PATCH 020/179] Test that if a corrupt wal-index header is encountered when attempting to commit a concurrent transaction, SQLITE_BUSY_SNAPSHOT is returned to the caller. FossilOrigin-Name: c746e0bd20cb136eed2b691f326657d266e2f1ed --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- test/concurrent2.test | 41 +++++++++++++++++++++++++++++++++++++++++ test/wal2.test | 37 ------------------------------------- test/wal_common.tcl | 40 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 90 insertions(+), 46 deletions(-) diff --git a/manifest b/manifest index b28ae7786e..57a2877c15 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\ssegfault\sthat\scould\soccur\sfollowing\san\sOOM\scondition\sin\sthe\sconcurrent\stransaction\scode. -D 2015-08-25T14:37:39.823 +C Test\sthat\sif\sa\scorrupt\swal-index\sheader\sis\sencountered\swhen\sattempting\sto\scommit\sa\sconcurrent\stransaction,\sSQLITE_BUSY_SNAPSHOT\sis\sreturned\sto\sthe\scaller. +D 2015-08-25T16:01:04.106 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -525,7 +525,7 @@ F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b F test/concfault.test 500f17c3fcfe7705114422bcc6ddd3c740001a43 F test/concurrent.test 26c2d49abbf4847ceed9bf8cf7fbe9a2a4ffc70c -F test/concurrent2.test 21a15630192ba92287070a1a58a6e497d393c55d +F test/concurrent2.test 4b9d1cc7126bb83bb08aca0e7fa10e0eab54b0c7 F test/concurrent3.test 7dcf81372c06cbac58e7e630aebf7292945947bb F test/conflict.test 841bcf7cabbfca39c577eb8411ea8601843b46a8 F test/conflict2.test 0d3af4fb534fa1bd020c79960bb56e4d52655f09 @@ -1262,7 +1262,7 @@ F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test ea8778d5b0df200adef2ca7c00c3c37d4375f772 F test/wal.test dbfc482e10c7263298833bb1fc60b3ac9d6340a1 -F test/wal2.test 1f841d2048080d32f552942e333fd99ce541dada +F test/wal2.test dc801f6d3f8f39d1534ffbd15a715b0b287508db F test/wal3.test 2b5445e5da44780b9b44712f5a38523f7aeb0941 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 88b5d9a6a3d1532497ee9f4296f010d66f07e33c @@ -1271,7 +1271,7 @@ F test/wal64k.test 163655ecd2cb8afef4737cac2a40fdd2eeaf20b8 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal8.test 75c42e1bc4545c277fed212f8fc9b7723cd02216 F test/wal9.test 378e76a9ad09cd9bee06c172ad3547b0129a6750 -F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe +F test/wal_common.tcl 37902864b63794e9bf59393b8887e21a9cbd4db8 F test/walbak.test b9f68e39646375c2b877be906babcc15d38b4877 F test/walbig.test f437473a16cfb314867c6b5d1dbcd519e73e3434 F test/walblock.test be48f3a75eff0b4456209f26b3ce186c2015497d @@ -1382,7 +1382,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3e7d6dd62dfa63d7def00bd00ac055a606a0c80d -R 7e8bf99bd9ee49e1baaabce6c0c30e59 +P 231b5880228cf01efe3981bc8be3150d79b422e5 +R f2d97f3b3defb10cf5a80dc03134f727 U dan -Z 16262c3600df8d68cd85cb26ef84be69 +Z ef5936cdf16a4565acadeed2265c73bc diff --git a/manifest.uuid b/manifest.uuid index 9e2c009bc9..203a63006a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -231b5880228cf01efe3981bc8be3150d79b422e5 \ No newline at end of file +c746e0bd20cb136eed2b691f326657d266e2f1ed \ No newline at end of file diff --git a/test/concurrent2.test b/test/concurrent2.test index d058733e48..f254e6d2aa 100644 --- a/test/concurrent2.test +++ b/test/concurrent2.test @@ -9,10 +9,13 @@ # #*********************************************************************** # +# Miscellaneous tests for transactions started with BEGIN CONCURRENT. +# set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/lock_common.tcl +source $testdir/wal_common.tcl set ::testprefix concurrent2 ifcapable !concurrent { @@ -296,5 +299,43 @@ do_multiclient_test tn { do_test 6.$tn.5 { sql3 { PRAGMA integrity_check } } {ok} do_test 6.$tn.5 { sql3 { SELECT count(*) from t1 } } {3} } + +#------------------------------------------------------------------------- +# Test that if a corrupt wal-index-header is encountered when attempting +# to commit a CONCURRENT transaction, the transaction is not committed +# (or rolled back) and that SQLITE_BUSY_SNAPSHOT is returned to the user. +# +catch { db close } +forcedelete test.db +testvfs tvfs +sqlite3 db test.db -vfs tvfs +do_execsql_test 7.1 { + PRAGMA journal_mode = wal; + BEGIN; + CREATE TABLE t1(a, b, PRIMARY KEY(a)); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + COMMIT; + BEGIN CONCURRENT; + INSERT INTO t1 VALUES(5, 6); + INSERT INTO t1 VALUES(7, 8); + SELECT * FROM t1; +} {wal 1 2 3 4 5 6 7 8} + +# Corrupt the wal-index header +incr_tvfs_hdr test.db 11 1 + +do_catchsql_test 7.2.1 { COMMIT } {1 {database is locked}} +do_test 7.2.2 { sqlite3_extended_errcode db } SQLITE_BUSY_SNAPSHOT + +do_execsql_test 7.3.1 { + SELECT * FROM t1; + ROLLBACK; +} {1 2 3 4 5 6 7 8} +do_execsql_test 7.3.2 { + SELECT * FROM t1; +} {1 2 3 4} + + finish_test diff --git a/test/wal2.test b/test/wal2.test index 9d45444d6a..ad2147c2bc 100644 --- a/test/wal2.test +++ b/test/wal2.test @@ -35,43 +35,6 @@ proc cond_incr_sync_count {adj} { } } -proc set_tvfs_hdr {file args} { - - # Set $nHdr to the number of bytes in the wal-index header: - set nHdr 48 - set nInt [expr {$nHdr/4}] - - if {[llength $args]>2} { - error {wrong # args: should be "set_tvfs_hdr fileName ?val1? ?val2?"} - } - - set blob [tvfs shm $file] - if {$::tcl_platform(byteOrder)=="bigEndian"} {set fmt I} {set fmt i} - - if {[llength $args]} { - set ia [lindex $args 0] - set ib $ia - if {[llength $args]==2} { - set ib [lindex $args 1] - } - binary scan $blob a[expr $nHdr*2]a* dummy tail - set blob [binary format ${fmt}${nInt}${fmt}${nInt}a* $ia $ib $tail] - tvfs shm $file $blob - } - - binary scan $blob ${fmt}${nInt} ints - return $ints -} - -proc incr_tvfs_hdr {file idx incrval} { - set ints [set_tvfs_hdr $file] - set v [lindex $ints $idx] - incr v $incrval - lset ints $idx $v - set_tvfs_hdr $file $ints -} - - #------------------------------------------------------------------------- # Test case wal2-1.*: # diff --git a/test/wal_common.tcl b/test/wal_common.tcl index 917ad598f6..e39bffdc26 100644 --- a/test/wal_common.tcl +++ b/test/wal_common.tcl @@ -90,4 +90,44 @@ proc wal_fix_walindex_cksum {hdrvar} { lset hdr 11 $c2 } +# This command assumes that $file is the name of a database file opened +# in wal mode using a [testvfs] VFS. It returns a list of the 12 32-bit +# integers that make up the wal-index-header for the named file. +# +proc set_tvfs_hdr {file args} { + + # Set $nHdr to the number of bytes in the wal-index header: + set nHdr 48 + set nInt [expr {$nHdr/4}] + + if {[llength $args]>2} { + error {wrong # args: should be "set_tvfs_hdr fileName ?val1? ?val2?"} + } + + set blob [tvfs shm $file] + if {$::tcl_platform(byteOrder)=="bigEndian"} {set fmt I} {set fmt i} + + if {[llength $args]} { + set ia [lindex $args 0] + set ib $ia + if {[llength $args]==2} { + set ib [lindex $args 1] + } + binary scan $blob a[expr $nHdr*2]a* dummy tail + set blob [binary format ${fmt}${nInt}${fmt}${nInt}a* $ia $ib $tail] + tvfs shm $file $blob + } + + binary scan $blob ${fmt}${nInt} ints + return $ints +} + +proc incr_tvfs_hdr {file idx incrval} { + set ints [set_tvfs_hdr $file] + set v [lindex $ints $idx] + incr v $incrval + lset ints $idx $v + set_tvfs_hdr $file $ints +} + From 57888f73007001d764dcb950cf5d862dbee30eb4 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 25 Aug 2015 17:16:33 +0000 Subject: [PATCH 021/179] If "PRAGMA integrity_check" is run while the database is being written by a CONCURRENT transaction, do not consider unreferenced pages to be an error. They may be part of the free-page list, which is not visible at the b-tree layer when running a CONCURRENT transaction. FossilOrigin-Name: f32b57b49311693eb0c0c9f6f14859e7b1fa93d8 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 8 +++++--- test/concurrent2.test | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 45 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 57a2877c15..8c41809276 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Test\sthat\sif\sa\scorrupt\swal-index\sheader\sis\sencountered\swhen\sattempting\sto\scommit\sa\sconcurrent\stransaction,\sSQLITE_BUSY_SNAPSHOT\sis\sreturned\sto\sthe\scaller. -D 2015-08-25T16:01:04.106 +C If\s"PRAGMA\sintegrity_check"\sis\srun\swhile\sthe\sdatabase\sis\sbeing\swritten\sby\sa\sCONCURRENT\stransaction,\sdo\snot\sconsider\sunreferenced\spages\sto\sbe\san\serror.\sThey\smay\sbe\spart\sof\sthe\sfree-page\slist,\swhich\sis\snot\svisible\sat\sthe\sb-tree\slayer\swhen\srunning\sa\sCONCURRENT\stransaction. +D 2015-08-25T17:16:33.362 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -279,7 +279,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 4d9134dc988a87838c06056c89c0e8c4700a0452 F src/bitvec.c d1f21d7d91690747881f03940584f4cc548c9d3d F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 -F src/btree.c a76329691503fc68ec374791aaf4074dc8686b3a +F src/btree.c aacef0cd0c57c2a1b2ed8a27794fc9e20b6e7a90 F src/btree.h 00d4cdb747c4172a5566faf037116985dbbc377e F src/btreeInt.h 171864bcd81635583dab7b8a04b19b454b18ef80 F src/build.c 1b5814e0eeaba096ae3ee17ebd139d2a25af7250 @@ -525,7 +525,7 @@ F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b F test/concfault.test 500f17c3fcfe7705114422bcc6ddd3c740001a43 F test/concurrent.test 26c2d49abbf4847ceed9bf8cf7fbe9a2a4ffc70c -F test/concurrent2.test 4b9d1cc7126bb83bb08aca0e7fa10e0eab54b0c7 +F test/concurrent2.test fa570bf9723f5c30fe40d9f2b1faa55c3c712c41 F test/concurrent3.test 7dcf81372c06cbac58e7e630aebf7292945947bb F test/conflict.test 841bcf7cabbfca39c577eb8411ea8601843b46a8 F test/conflict2.test 0d3af4fb534fa1bd020c79960bb56e4d52655f09 @@ -1382,7 +1382,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 231b5880228cf01efe3981bc8be3150d79b422e5 -R f2d97f3b3defb10cf5a80dc03134f727 +P c746e0bd20cb136eed2b691f326657d266e2f1ed +R e20e0a06f63966f8ec3b7d9acf7660a9 U dan -Z ef5936cdf16a4565acadeed2265c73bc +Z 220f77062430ec96f6fa5ad5d41eac8b diff --git a/manifest.uuid b/manifest.uuid index 203a63006a..5bcd4bc5e7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c746e0bd20cb136eed2b691f326657d266e2f1ed \ No newline at end of file +f32b57b49311693eb0c0c9f6f14859e7b1fa93d8 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 9b90f58d07..70c36a933c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -9653,9 +9653,11 @@ char *sqlite3BtreeIntegrityCheck( } pBt->db->flags = savedDbFlags; - /* Make sure every page in the file is referenced - */ - for(i=1; i<=sCheck.nPage && sCheck.mxErr; i++){ + /* Make sure every page in the file is referenced. Skip this if the + ** database is currently being written by a CONCURRENT transaction (it + ** may fail as pages that were part of the free-list when the transaction + ** was opened cannot be counted). */ + for(i=1; ISCONCURRENT==0 && i<=sCheck.nPage && sCheck.mxErr; i++){ #ifdef SQLITE_OMIT_AUTOVACUUM if( getPageReferenced(&sCheck, i)==0 ){ checkAppendMsg(&sCheck, "Page %d is never used", i); diff --git a/test/concurrent2.test b/test/concurrent2.test index f254e6d2aa..3472db61f4 100644 --- a/test/concurrent2.test +++ b/test/concurrent2.test @@ -336,6 +336,38 @@ do_execsql_test 7.3.2 { SELECT * FROM t1; } {1 2 3 4} +#------------------------------------------------------------------------- +# Test that "PRAGMA integrity_check" works within a concurrent +# transaction. Within a concurrent transaction, "PRAGMA integrity_check" +# is unable to detect unused database pages, but can detect other types +# of corruption. +# +reset_db +do_execsql_test 8.1 { + PRAGMA journal_mode = wal; + CREATE TABLE kv(k INTEGER PRIMARY KEY, v UNIQUE); + INSERT INTO kv VALUES(NULL, randomblob(750)); + INSERT INTO kv SELECT NULL, randomblob(750) FROM kv; + INSERT INTO kv SELECT NULL, randomblob(750) FROM kv; + INSERT INTO kv SELECT NULL, randomblob(750) FROM kv; + INSERT INTO kv SELECT NULL, randomblob(750) FROM kv; + INSERT INTO kv SELECT NULL, randomblob(750) FROM kv; + DELETE FROM kv WHERE rowid%2; + PRAGMA freelist_count; +} {wal 33} +do_execsql_test 8.2 { PRAGMA integrity_check } ok +do_execsql_test 8.3 { + BEGIN CONCURRENT; + PRAGMA integrity_check; +} {ok} +do_execsql_test 8.4 { + INSERT INTO kv VALUES(1100, 1100); + PRAGMA integrity_check; +} {ok} +do_execsql_test 8.5 { + COMMIT; + PRAGMA integrity_check; +} {ok} finish_test From fef3410f7ff75d88a478c221495017f0fbae2e82 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 25 Aug 2015 19:10:29 +0000 Subject: [PATCH 022/179] Add miscellaneous test cases for concurrent transactions. FossilOrigin-Name: 779b1d0e17bc54062b2b09cdbf94e9e2f4bae4f7 --- manifest | 14 +++++----- manifest.uuid | 2 +- test/concurrent.test | 58 +++++++++++++++++++++++++++++++++++++++++- test/concurrent2.test | 59 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 124 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 8c41809276..452016bd91 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\s"PRAGMA\sintegrity_check"\sis\srun\swhile\sthe\sdatabase\sis\sbeing\swritten\sby\sa\sCONCURRENT\stransaction,\sdo\snot\sconsider\sunreferenced\spages\sto\sbe\san\serror.\sThey\smay\sbe\spart\sof\sthe\sfree-page\slist,\swhich\sis\snot\svisible\sat\sthe\sb-tree\slayer\swhen\srunning\sa\sCONCURRENT\stransaction. -D 2015-08-25T17:16:33.362 +C Add\smiscellaneous\stest\scases\sfor\sconcurrent\stransactions. +D 2015-08-25T19:10:29.114 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -524,8 +524,8 @@ F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b F test/concfault.test 500f17c3fcfe7705114422bcc6ddd3c740001a43 -F test/concurrent.test 26c2d49abbf4847ceed9bf8cf7fbe9a2a4ffc70c -F test/concurrent2.test fa570bf9723f5c30fe40d9f2b1faa55c3c712c41 +F test/concurrent.test ecf97fdcfb11dda1db52b2714d7d52d0922789f1 +F test/concurrent2.test de43cd6703360dc6268907f1617f0d353d8a43c1 F test/concurrent3.test 7dcf81372c06cbac58e7e630aebf7292945947bb F test/conflict.test 841bcf7cabbfca39c577eb8411ea8601843b46a8 F test/conflict2.test 0d3af4fb534fa1bd020c79960bb56e4d52655f09 @@ -1382,7 +1382,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c746e0bd20cb136eed2b691f326657d266e2f1ed -R e20e0a06f63966f8ec3b7d9acf7660a9 +P f32b57b49311693eb0c0c9f6f14859e7b1fa93d8 +R cbe24c6dda59af48460d1125c5ffff88 U dan -Z 220f77062430ec96f6fa5ad5d41eac8b +Z 967a64b7790792e3ecfbe18cee7c81a2 diff --git a/manifest.uuid b/manifest.uuid index 5bcd4bc5e7..16d88ba06c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f32b57b49311693eb0c0c9f6f14859e7b1fa93d8 \ No newline at end of file +779b1d0e17bc54062b2b09cdbf94e9e2f4bae4f7 \ No newline at end of file diff --git a/test/concurrent.test b/test/concurrent.test index cf7d59257c..4bc2852c60 100644 --- a/test/concurrent.test +++ b/test/concurrent.test @@ -417,7 +417,7 @@ do_multiclient_test tn { } #------------------------------------------------------------------------- -# Unlocked transactions may not modify the user_version or application_id. +# Concurrent transactions may not modify the user_version or application_id. # reset_db do_execsql_test 3.0 { @@ -448,6 +448,62 @@ do_execsql_test 3.5 { SELECT * FROM t1; } {10 0 a b c d} +#------------------------------------------------------------------------- +# However, another transaction modifying the user_version or application_id +# should not cause a conflict. And committing a concurrent transaction does not +# clobber the modification - even if the concurrent transaction allocates or +# frees database pages. +# +do_multiclient_test tn { + do_test 4.$tn.1 { + sql1 { + PRAGMA journal_mode = wal; + CREATE TABLE ttt(y UNIQUE, z UNIQUE); + PRAGMA user_version = 14; + BEGIN CONCURRENT; + INSERT INTO ttt VALUES('y', 'z'); + } + } {wal} + do_test 4.$tn.2 { + sql2 { PRAGMA user_version = 16 } + sql1 COMMIT + sql1 { PRAGMA user_version } + } {16} + + do_test 4.$tn.3 { + sql1 { + BEGIN CONCURRENT; + INSERT INTO ttt VALUES(randomblob(10000), randomblob(4)); + PRAGMA user_version; + } + } {16} + do_test 4.$tn.4 { + sql2 { PRAGMA user_version = 1234 } + sql1 { + PRAGMA user_version; + COMMIT; + PRAGMA user_version; + PRAGMA integrity_check; + } + } {16 1234 ok} + + do_test 4.$tn.5 { + sql1 { + BEGIN CONCURRENT; + DELETE FROM ttt; + PRAGMA user_version; + } + } {1234} + do_test 4.$tn.4 { + sql2 { PRAGMA user_version = 5678 } + sql1 { + PRAGMA user_version; + COMMIT; + PRAGMA user_version; + PRAGMA integrity_check; + } + } {1234 5678 ok} +} finish_test diff --git a/test/concurrent2.test b/test/concurrent2.test index 3472db61f4..7f04a06b31 100644 --- a/test/concurrent2.test +++ b/test/concurrent2.test @@ -369,5 +369,64 @@ do_execsql_test 8.5 { PRAGMA integrity_check; } {ok} +#------------------------------------------------------------------------- +# Test that concurrent transactions do not allow foreign-key constraints +# to be bypassed. +# +do_multiclient_test tn { + do_test 9.$tn.1 { + sql1 { + PRAGMA journal_mode = wal; + CREATE TABLE pp(i INTEGER PRIMARY KEY, j); + CREATE TABLE cc(a, b REFERENCES pp); + + WITH seq(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM seq WHERE i<100) + INSERT INTO pp SELECT i, randomblob(1000) FROM seq; + + PRAGMA foreign_keys = 1; + } + } {wal} + + + do_test 9.$tn.2.1 { + sql1 { + BEGIN CONCURRENT; + INSERT INTO cc VALUES(42, 42); + } + } {} + do_test 9.$tn.2.2 { + sql2 { DELETE FROM pp WHERE i=42 } + list [catch { sql1 COMMIT } msg] $msg + } {1 {database is locked}} + do_test 9.$tn.2.3 { + sql1 ROLLBACK + } {} + + do_test 9.$tn.3.1 { + sql1 { + PRAGMA foreign_keys = 0; + BEGIN CONCURRENT; + INSERT INTO cc VALUES(43, 43); + } + } {} + do_test 9.$tn.3.2 { + sql2 { DELETE FROM pp WHERE i=43 } + list [catch { sql1 COMMIT } msg] $msg + } {0 {}} + + do_test 9.$tn.4.1 { + sql1 { + PRAGMA foreign_keys = on; + BEGIN CONCURRENT; + INSERT INTO cc VALUES(44, 44); + } + } {} + do_test 9.$tn.4.2 { + sql2 { DELETE FROM pp WHERE i=1 } + list [catch { sql1 COMMIT } msg] $msg + } {0 {}} +} + + finish_test From de36c76a2369c76d6edf53f11381357685dff889 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 26 Aug 2015 18:02:20 +0000 Subject: [PATCH 023/179] Fix a problem allowing some conflicting transactions to be committed. FossilOrigin-Name: a0566382d564ca17fd13475a44fed8f714742d97 --- manifest | 14 +++---- manifest.uuid | 2 +- src/wal.c | 2 +- test/concurrent3.test | 97 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 452016bd91..79ab2b14bc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\smiscellaneous\stest\scases\sfor\sconcurrent\stransactions. -D 2015-08-25T19:10:29.114 +C Fix\sa\sproblem\sallowing\ssome\sconflicting\stransactions\sto\sbe\scommitted. +D 2015-08-26T18:02:20.162 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -411,7 +411,7 @@ F src/vdbesort.c f5009e7a35e3065635d8918b9a31f498a499976b F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 F src/vtab.c d31174e4c8f592febab3fa7f69e18320b4fd657a F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb -F src/wal.c a21412a803f0eafecf48e707fb97c64368a0d267 +F src/wal.c 44ec009f7742a660d3558845e47e870af4542689 F src/wal.h 903ef67e17f8b466dc7cfc4186fc23e80be10ff8 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba F src/where.c 66518a14a1238611aa0744d6980b6b7f544f4816 @@ -526,7 +526,7 @@ F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b F test/concfault.test 500f17c3fcfe7705114422bcc6ddd3c740001a43 F test/concurrent.test ecf97fdcfb11dda1db52b2714d7d52d0922789f1 F test/concurrent2.test de43cd6703360dc6268907f1617f0d353d8a43c1 -F test/concurrent3.test 7dcf81372c06cbac58e7e630aebf7292945947bb +F test/concurrent3.test 8474b7ac80bc977bab4fe014c0b036c16779d8cb F test/conflict.test 841bcf7cabbfca39c577eb8411ea8601843b46a8 F test/conflict2.test 0d3af4fb534fa1bd020c79960bb56e4d52655f09 F test/conflict3.test dec0634c0f31dec9a4b01c63063e939f0cd21b6b @@ -1382,7 +1382,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f32b57b49311693eb0c0c9f6f14859e7b1fa93d8 -R cbe24c6dda59af48460d1125c5ffff88 +P 779b1d0e17bc54062b2b09cdbf94e9e2f4bae4f7 +R 522c73a54721e2428308711f9566b92e U dan -Z 967a64b7790792e3ecfbe18cee7c81a2 +Z 5afacf522fa16d68d46082708ed9d148 diff --git a/manifest.uuid b/manifest.uuid index 16d88ba06c..5d4bfea3dc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -779b1d0e17bc54062b2b09cdbf94e9e2f4bae4f7 \ No newline at end of file +a0566382d564ca17fd13475a44fed8f714742d97 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 4db4209750..445d084628 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2680,7 +2680,7 @@ int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPage1, Bitvec *pAllRead){ int iMax = (iHash==0) ? HASHTABLE_NPAGE_ONE : HASHTABLE_NPAGE; if( iMin<1 ) iMin = 1; if( iMax>head.mxFrame ) iMax = head.mxFrame; - for(i=iMin; i<=iMax; i++){ + for(i=iMin; rc==SQLITE_OK && i<=iMax; i++){ PgHdr *pPg; if( aPgno[i]==1 ){ /* Check that the schema cookie has not been modified. If diff --git a/test/concurrent3.test b/test/concurrent3.test index 25e5d7c6c9..a90fa2b973 100644 --- a/test/concurrent3.test +++ b/test/concurrent3.test @@ -24,6 +24,12 @@ ifcapable !concurrent { return } +db close +sqlite3_shutdown +#test_sqlite3_log xLog +#proc xLog {error_code msg} { puts "$error_code: $msg" } +reset_db + proc create_schema {} { db eval { PRAGMA journal_mode = wal; @@ -128,5 +134,96 @@ foreach {tn oplist} { } } +#------------------------------------------------------------------------- +# +proc create_schema2 {} { + db eval { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x INTEGER PRIMARY KEY, y); + CREATE INDEX i1 ON t1(y); + } +} + +proc randint {nMax} { + db eval {SELECT abs(random() % $nMax)} +} + +proc do_sql_op2 {db iOp} { + switch -- $iOp { + i { + # Insert 1 rows. + set r [randint 1000000000] + set ::rows($r) 1 + #puts "insert row $r" + $db eval { INSERT OR IGNORE INTO t1 VALUES($r, randomblob(50)); } + } + + d { + # Insert 1 row + set keys [array names ::rows] + set r [randint [llength $keys]] + set rowid [lindex $keys $r] + $db eval { DELETE FROM t1 WHERE x=$rowid } + unset ::rows($rowid) + } + } +} + +foreach {tn nRepeat oplist} { + - - ---------------------------- + 1 100 { 1iiiiiiiiii } + 2 100 { 1i 2d } + 3 100 { 1d 2i } + 4 50 { 1d 2i 3d } + 5 500 { 1i 2i 3i 4i } +} { + if {[string range $oplist 0 0]=="-"} { + array unset rows + reset_db + create_schema2 + continue + } + + foreach db $DBLIST { + sqlite3 $db test.db + set stats($db,0) 0 + set stats($db,1) 0 + } + array unset used + + do_test 2.$tn { + + for {set i 0} {$i < $nRepeat} {incr i} { + foreach db $DBLIST { $db eval "BEGIN CONCURRENT" } + + foreach op $oplist { + set iDb [string range $op 0 0] + set used(db$iDb) 1 + foreach char [split [string range $op 1 end] {}] { + do_sql_op2 "db$iDb" $char + } + } + + foreach db $DBLIST { + if {$i==272 && $db=="db4"} breakpoint + set rc [catch { $db eval COMMIT } msg] + if {$rc} { $db eval ROLLBACK } + incr stats($db,$rc) + } + set res [db eval {PRAGMA integrity_check}] + if {$res != "ok"} { puts "after $db $rc: $res" ; after 1000000 } + } + } {} + + foreach db $DBLIST { + $db close + } + foreach k [lsort [array names used]] { + puts "$k: $stats($k,0) committed, $stats($k,1) rolled back" + } +} + + + finish_test From f6cf5ea790764b524c3339a1ba1d06be00f1ae3f Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 26 Aug 2015 18:54:45 +0000 Subject: [PATCH 024/179] Fix an assert() in pager.c that could fail in a concurrent transaction. FossilOrigin-Name: 69394ddaa2bc9d26477b4359c676c598b733ac9f --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/pager.c | 2 +- test/concurrent.test | 21 +++++++++++++++++++++ test/concurrent3.test | 2 +- 5 files changed, 32 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 79ab2b14bc..4792a3a28f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sallowing\ssome\sconflicting\stransactions\sto\sbe\scommitted. -D 2015-08-26T18:02:20.162 +C Fix\san\sassert()\sin\spager.c\sthat\scould\sfail\sin\sa\sconcurrent\stransaction. +D 2015-08-26T18:54:45.787 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -324,7 +324,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 388c023582b17890f10c980b30ec1922b471753b F src/os_win.c 40b3af7a47eb1107d0d69e592bec345a3b7b798a F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 2e2559e64e825e39c033c0744237733cec70d636 +F src/pager.c bb8e237a54d162a6cc22503f71895851dae9a0bb F src/pager.h 1335b624cd540815c8c977172589d208d1c251a6 F src/parse.y 1e645cacb93979c59f2a510ee2c100e769bd5e3c F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0 @@ -524,9 +524,9 @@ F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b F test/concfault.test 500f17c3fcfe7705114422bcc6ddd3c740001a43 -F test/concurrent.test ecf97fdcfb11dda1db52b2714d7d52d0922789f1 +F test/concurrent.test 634b6a88f1942f5d68cc89d4d5efa2b11ba7913c F test/concurrent2.test de43cd6703360dc6268907f1617f0d353d8a43c1 -F test/concurrent3.test 8474b7ac80bc977bab4fe014c0b036c16779d8cb +F test/concurrent3.test 0a5f7e3036d1eccf0782d7153ac21f5f222e9468 F test/conflict.test 841bcf7cabbfca39c577eb8411ea8601843b46a8 F test/conflict2.test 0d3af4fb534fa1bd020c79960bb56e4d52655f09 F test/conflict3.test dec0634c0f31dec9a4b01c63063e939f0cd21b6b @@ -1382,7 +1382,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 779b1d0e17bc54062b2b09cdbf94e9e2f4bae4f7 -R 522c73a54721e2428308711f9566b92e +P a0566382d564ca17fd13475a44fed8f714742d97 +R 82cb8297df85a5e2338dab90f8e6448e U dan -Z 5afacf522fa16d68d46082708ed9d148 +Z 29e210082fd7b7d5f83c5decd49ba489 diff --git a/manifest.uuid b/manifest.uuid index 5d4bfea3dc..fc6505589c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a0566382d564ca17fd13475a44fed8f714742d97 \ No newline at end of file +69394ddaa2bc9d26477b4359c676c598b733ac9f \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index ecdad403dc..9e6c2a0b9c 100644 --- a/src/pager.c +++ b/src/pager.c @@ -906,7 +906,7 @@ static int assert_pager_state(Pager *p){ if( !pagerUseWal(pPager) ){ assert( p->eLock>=RESERVED_LOCK ); } - assert( pPager->dbSize==pPager->dbOrigSize ); + assert( pPager->dbSize==pPager->dbOrigSize || pPager->pAllRead ); assert( pPager->dbOrigSize==pPager->dbFileSize ); assert( pPager->dbOrigSize==pPager->dbHintSize ); assert( pPager->setMaster==0 ); diff --git a/test/concurrent.test b/test/concurrent.test index 4bc2852c60..4e62d9e33f 100644 --- a/test/concurrent.test +++ b/test/concurrent.test @@ -505,5 +505,26 @@ do_multiclient_test tn { } {1234 5678 ok} } +do_multiclient_test tn { + do_test 5.$tn.1 { + sql1 { + PRAGMA journal_mode = wal; + CREATE TABLE tt(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b); + INSERT INTO tt VALUES(1, randomblob(400)); + BEGIN CONCURRENT; + } + } {wal} + + do_test 5.$tn.2 { + sql1 { UPDATE t2 SET b=5 WHERE a=3 } + sql2 { INSERT INTO tt VALUES(2, randomblob(6000)) } + } {} + + do_test 5.$tn.3 { + sql1 { COMMIT } + } {} +} + finish_test diff --git a/test/concurrent3.test b/test/concurrent3.test index a90fa2b973..3ad6a1cce4 100644 --- a/test/concurrent3.test +++ b/test/concurrent3.test @@ -176,6 +176,7 @@ foreach {tn nRepeat oplist} { 3 100 { 1d 2i } 4 50 { 1d 2i 3d } 5 500 { 1i 2i 3i 4i } + 6 500 { 1i 2d 3d 4d } } { if {[string range $oplist 0 0]=="-"} { array unset rows @@ -205,7 +206,6 @@ foreach {tn nRepeat oplist} { } foreach db $DBLIST { - if {$i==272 && $db=="db4"} breakpoint set rc [catch { $db eval COMMIT } msg] if {$rc} { $db eval ROLLBACK } incr stats($db,$rc) From 987f821f791502cc5ccb83caa8bf399427c4d94d Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 27 Aug 2015 17:42:38 +0000 Subject: [PATCH 025/179] Fix a problem whereby concurrent transactions would not consider pages read by the transaction before the first write statement. FossilOrigin-Name: fc17f73170a27c2fe511ed6b6d488535c4e35bae --- manifest | 18 +++++++-------- manifest.uuid | 2 +- src/btree.c | 46 +++++++++++++++++++++----------------- src/pager.c | 51 +++++++++++++++++++++---------------------- src/pager.h | 10 +++++++-- test/concurrent2.test | 25 +++++++++++++++++++++ 6 files changed, 94 insertions(+), 58 deletions(-) diff --git a/manifest b/manifest index 4792a3a28f..4440e87c40 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sassert()\sin\spager.c\sthat\scould\sfail\sin\sa\sconcurrent\stransaction. -D 2015-08-26T18:54:45.787 +C Fix\sa\sproblem\swhereby\sconcurrent\stransactions\swould\snot\sconsider\spages\sread\sby\sthe\stransaction\sbefore\sthe\sfirst\swrite\sstatement. +D 2015-08-27T17:42:38.260 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -279,7 +279,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 4d9134dc988a87838c06056c89c0e8c4700a0452 F src/bitvec.c d1f21d7d91690747881f03940584f4cc548c9d3d F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 -F src/btree.c aacef0cd0c57c2a1b2ed8a27794fc9e20b6e7a90 +F src/btree.c 88ff0f5bdcd200e4b8cdc2f49a89369112a592d7 F src/btree.h 00d4cdb747c4172a5566faf037116985dbbc377e F src/btreeInt.h 171864bcd81635583dab7b8a04b19b454b18ef80 F src/build.c 1b5814e0eeaba096ae3ee17ebd139d2a25af7250 @@ -324,8 +324,8 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 388c023582b17890f10c980b30ec1922b471753b F src/os_win.c 40b3af7a47eb1107d0d69e592bec345a3b7b798a F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c bb8e237a54d162a6cc22503f71895851dae9a0bb -F src/pager.h 1335b624cd540815c8c977172589d208d1c251a6 +F src/pager.c 732f3b107ac7180e4c628114a5d06dda00393080 +F src/pager.h f00930ca3bfc0f3298b08a69ed7b920e89b531de F src/parse.y 1e645cacb93979c59f2a510ee2c100e769bd5e3c F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0 F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9 @@ -525,7 +525,7 @@ F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b F test/concfault.test 500f17c3fcfe7705114422bcc6ddd3c740001a43 F test/concurrent.test 634b6a88f1942f5d68cc89d4d5efa2b11ba7913c -F test/concurrent2.test de43cd6703360dc6268907f1617f0d353d8a43c1 +F test/concurrent2.test d42aaa1d0aaf2c41c8d5de204962b125b411557d F test/concurrent3.test 0a5f7e3036d1eccf0782d7153ac21f5f222e9468 F test/conflict.test 841bcf7cabbfca39c577eb8411ea8601843b46a8 F test/conflict2.test 0d3af4fb534fa1bd020c79960bb56e4d52655f09 @@ -1382,7 +1382,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a0566382d564ca17fd13475a44fed8f714742d97 -R 82cb8297df85a5e2338dab90f8e6448e +P 69394ddaa2bc9d26477b4359c676c598b733ac9f +R aacab69ab35db4167de1724271774606 U dan -Z 29e210082fd7b7d5f83c5decd49ba489 +Z d9ef073e0b6bc6533e113c00c2daa9fc diff --git a/manifest.uuid b/manifest.uuid index fc6505589c..72a90a6e05 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -69394ddaa2bc9d26477b4359c676c598b733ac9f \ No newline at end of file +fc17f73170a27c2fe511ed6b6d488535c4e35bae \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 70c36a933c..9986c95325 100644 --- a/src/btree.c +++ b/src/btree.c @@ -595,15 +595,16 @@ static void btreePtrmapEnd(BtShared *pBt, int op, int iSvpt){ */ static int btreePtrmapAllocate(BtShared *pBt){ int rc = SQLITE_OK; - BtreePtrmap *pMap = sqlite3_malloc(sizeof(BtreePtrmap)); - assert( pBt->pMap==0 && sqlite3PagerIsConcurrent(pBt->pPager) ); - if( pMap==0 ){ - rc = SQLITE_NOMEM; - }else{ - memset(&pBt->pPage1->aData[32], 0, sizeof(u32)*2); - memset(pMap, 0, sizeof(BtreePtrmap)); - pMap->iFirst = pBt->nPage + 1; - pBt->pMap = pMap; + if( pBt->pMap==0 ){ + BtreePtrmap *pMap = sqlite3_malloc(sizeof(BtreePtrmap)); + if( pMap==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(&pBt->pPage1->aData[32], 0, sizeof(u32)*2); + memset(pMap, 0, sizeof(BtreePtrmap)); + pMap->iFirst = pBt->nPage + 1; + pBt->pMap = pMap; + } } return rc; } @@ -3273,6 +3274,7 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ sqlite3 *pBlock = 0; BtShared *pBt = p->pBt; int rc = SQLITE_OK; + int bConcurrent = (p->db->bConcurrent && !ISAUTOVACUUM); sqlite3BtreeEnter(p); btreeIntegrity(p); @@ -3339,18 +3341,11 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ if( (pBt->btsFlags & BTS_READ_ONLY)!=0 ){ rc = SQLITE_READONLY; }else{ - int exFlag = (p->db->bConcurrent && !ISAUTOVACUUM) ? -1 : (wrflag>1); - int bSubjInMem = sqlite3TempInMemory(p->db); - assert( p->db->bConcurrent==0 || wrflag==1 ); - rc = sqlite3PagerBegin(pBt->pPager, exFlag, bSubjInMem); + int exFlag = bConcurrent ? -1 : (wrflag>1); + rc = sqlite3PagerBegin(pBt->pPager, exFlag, sqlite3TempInMemory(p->db)); if( rc==SQLITE_OK ){ rc = newDatabase(pBt); } -#ifdef SQLITE_ENABLE_CONCURRENT - if( rc==SQLITE_OK && sqlite3PagerIsConcurrent(pBt->pPager) ){ - rc = btreePtrmapAllocate(pBt); - } -#endif } } @@ -3402,6 +3397,15 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ trans_begun: +#ifdef SQLITE_ENABLE_CONCURRENT + if( bConcurrent && rc==SQLITE_OK && sqlite3PagerIsWal(pBt->pPager) ){ + rc = sqlite3PagerBeginConcurrent(pBt->pPager); + if( rc==SQLITE_OK && wrflag ){ + rc = btreePtrmapAllocate(pBt); + } + } +#endif + if( rc==SQLITE_OK && wrflag ){ /* This call makes sure that the pager has the correct number of ** open savepoints. If the second parameter is greater than 0 and @@ -3937,7 +3941,6 @@ static int btreeFixUnlocked(Btree *p){ Pgno nPage = btreePagecount(pBt); u32 nFree = get4byte(&p1[36]); - assert( sqlite3PagerIsConcurrent(pPager) ); assert( pBt->pMap ); rc = sqlite3PagerUpgradeSnapshot(pPager, pPage1->pDbPage); assert( p1==pPage1->aData ); @@ -4101,8 +4104,11 @@ static void btreeEndTransaction(Btree *p){ unlockBtreeIfUnused(pBt); } - /* If this was an CONCURRENT transaction, delete the pBt->pMap object */ + /* If this was an CONCURRENT transaction, delete the pBt->pMap object. + ** Also call PagerEndConcurrent() to ensure that the pager has discarded + ** the record of all pages read within the transaction. */ btreePtrmapDelete(pBt); + sqlite3PagerEndConcurrent(pBt->pPager); btreeIntegrity(p); } diff --git a/src/pager.c b/src/pager.c index 9e6c2a0b9c..7db7544552 100644 --- a/src/pager.c +++ b/src/pager.c @@ -1741,16 +1741,35 @@ static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ return rc; } +#ifdef SQLITE_ENABLE_CONCURRENT +int sqlite3PagerBeginConcurrent(Pager *pPager){ + int rc = SQLITE_OK; + if( pPager->pAllRead==0 ){ + pPager->pAllRead = sqlite3BitvecCreate(pPager->dbSize); + if( pPager->pAllRead==0 ){ + rc = SQLITE_NOMEM; + } + } + return rc; +} + +void sqlite3PagerEndConcurrent(Pager *pPager){ + sqlite3BitvecDestroy(pPager->pAllRead); + pPager->pAllRead = 0; +} + +int sqlite3PagerIsWal(Pager *pPager){ + return pPager->pWal!=0; +} +#endif + /* ** Free the Pager.pInJournal and Pager.pAllRead bitvec objects. */ static void pagerFreeBitvecs(Pager *pPager){ sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; -#ifdef SQLITE_ENABLE_CONCURRENT - sqlite3BitvecDestroy(pPager->pAllRead); - pPager->pAllRead = 0; -#endif + sqlite3PagerEndConcurrent(pPager); } /* @@ -5612,10 +5631,6 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ if( ALWAYS(pPager->eState==PAGER_READER) ){ assert( pPager->pInJournal==0 ); -#ifdef SQLITE_ENABLE_CONCURRENT - assert( pPager->pAllRead==0 ); -#endif - if( pagerUseWal(pPager) ){ /* If the pager is configured to use locking_mode=exclusive, and an ** exclusive lock on the database is not already held, obtain it now. @@ -5631,17 +5646,8 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ /* Grab the write lock on the log file. If successful, upgrade to ** PAGER_RESERVED state. Otherwise, return an error code to the caller. ** The busy-handler is not invoked if another connection already - ** holds the write-lock. If possible, the upper layer will call it. - */ -#ifdef SQLITE_ENABLE_CONCURRENT - if( exFlag<0 ){ - pPager->pAllRead = sqlite3BitvecCreate(pPager->dbSize); - if( pPager->pAllRead==0 ){ - rc = SQLITE_NOMEM; - } - }else -#endif - { + ** holds the write-lock. If possible, the upper layer will call it. */ + if( exFlag>=0 ){ rc = sqlite3WalBeginWriteTransaction(pPager->pWal); } }else{ @@ -6184,13 +6190,6 @@ void sqlite3PagerSetDbsize(Pager *pPager, Pgno nSz){ pPager->dbSize = nSz; } -/* -** Return true if this pager is currently within an CONCURRENT transaction. -*/ -int sqlite3PagerIsConcurrent(Pager *pPager){ - return pPager->pAllRead!=0; -} - /* ** If this is a WAL mode connection and the WRITER lock is currently held, ** relinquish it. diff --git a/src/pager.h b/src/pager.h index c9ebaf1116..1df91bbca0 100644 --- a/src/pager.h +++ b/src/pager.h @@ -194,11 +194,17 @@ void sqlite3PagerTruncateImage(Pager*,Pgno); void sqlite3PagerRekey(DbPage*, Pgno, u16); +#ifdef SQLITE_ENABLE_CONCURRENT +void sqlite3PagerEndConcurrent(Pager*); +int sqlite3PagerBeginConcurrent(Pager*); void sqlite3PagerDropExclusiveLock(Pager*); -int sqlite3PagerIsConcurrent(Pager*); -int sqlite3PagerIswriteable(DbPage*); int sqlite3PagerUpgradeSnapshot(Pager *pPager, DbPage*); void sqlite3PagerSetDbsize(Pager *pPager, Pgno); +#else +# define sqlite3PagerEndConcurrent(x) +#endif + +int sqlite3PagerIswriteable(DbPage*); #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) void *sqlite3PagerCodec(DbPage *); diff --git a/test/concurrent2.test b/test/concurrent2.test index 7f04a06b31..8650caea20 100644 --- a/test/concurrent2.test +++ b/test/concurrent2.test @@ -427,6 +427,31 @@ do_multiclient_test tn { } {0 {}} } +do_multiclient_test tn { + do_test 10.$tn.1 { + sql1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(a); + CREATE TABLE t2(b); + INSERT INTO t1 VALUES(1), (2), (3); + INSERT INTO t2 VALUES(1), (2), (3); + } + } {wal} + + do_test 10.$tn.2 { + sql1 { + BEGIN CONCURRENT; + SELECT * FROM t1; + INSERT INTO t2 VALUES(4); + } + } {1 2 3} + + do_test 10.$tn.3 { + sql2 { INSERT INTO t1 VALUES(4) } + list [catch {sql1 COMMIT} msg] $msg + } {1 {database is locked}} +} + finish_test From 6b3e51bd330c9510695fbbd6443ab3f7a5e3d029 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 27 Aug 2015 19:22:56 +0000 Subject: [PATCH 026/179] Add test cases for concurrent transactions and long-lived SELECT statements. FossilOrigin-Name: fd4798cb7af263409c20d3cf81236b830bd68570 --- manifest | 12 ++--- manifest.uuid | 2 +- test/concurrent2.test | 102 ++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 106 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 4440e87c40..a9bcad179f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swhereby\sconcurrent\stransactions\swould\snot\sconsider\spages\sread\sby\sthe\stransaction\sbefore\sthe\sfirst\swrite\sstatement. -D 2015-08-27T17:42:38.260 +C Add\stest\scases\sfor\sconcurrent\stransactions\sand\slong-lived\sSELECT\sstatements. +D 2015-08-27T19:22:56.303 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -525,7 +525,7 @@ F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b F test/concfault.test 500f17c3fcfe7705114422bcc6ddd3c740001a43 F test/concurrent.test 634b6a88f1942f5d68cc89d4d5efa2b11ba7913c -F test/concurrent2.test d42aaa1d0aaf2c41c8d5de204962b125b411557d +F test/concurrent2.test f20913d376d993a85f28ef3323f09d4cba37206a F test/concurrent3.test 0a5f7e3036d1eccf0782d7153ac21f5f222e9468 F test/conflict.test 841bcf7cabbfca39c577eb8411ea8601843b46a8 F test/conflict2.test 0d3af4fb534fa1bd020c79960bb56e4d52655f09 @@ -1382,7 +1382,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 69394ddaa2bc9d26477b4359c676c598b733ac9f -R aacab69ab35db4167de1724271774606 +P fc17f73170a27c2fe511ed6b6d488535c4e35bae +R 54233c250c37b5d4c5cad57267aa2abd U dan -Z d9ef073e0b6bc6533e113c00c2daa9fc +Z 1ef37227dc1b9c5caa6e609be7886347 diff --git a/manifest.uuid b/manifest.uuid index 72a90a6e05..55734e47ba 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fc17f73170a27c2fe511ed6b6d488535c4e35bae \ No newline at end of file +fd4798cb7af263409c20d3cf81236b830bd68570 \ No newline at end of file diff --git a/test/concurrent2.test b/test/concurrent2.test index 8650caea20..aa2cd62217 100644 --- a/test/concurrent2.test +++ b/test/concurrent2.test @@ -427,18 +427,25 @@ do_multiclient_test tn { } {0 {}} } +#------------------------------------------------------------------------- +# Test that even if a SELECT statement appears before all writes within +# a CONCURRENT transaction, the pages it reads are still considered when +# considering whether or not the transaction may be committed. +# do_multiclient_test tn { - do_test 10.$tn.1 { + do_test 10.$tn.1.1 { sql1 { PRAGMA journal_mode = wal; CREATE TABLE t1(a); CREATE TABLE t2(b); + CREATE TABLE t3(c); INSERT INTO t1 VALUES(1), (2), (3); INSERT INTO t2 VALUES(1), (2), (3); + INSERT INTO t3 VALUES(1), (2), (3); } } {wal} - do_test 10.$tn.2 { + do_test 10.$tn.1.2 { sql1 { BEGIN CONCURRENT; SELECT * FROM t1; @@ -446,10 +453,99 @@ do_multiclient_test tn { } } {1 2 3} - do_test 10.$tn.3 { + do_test 10.$tn.1.3 { sql2 { INSERT INTO t1 VALUES(4) } list [catch {sql1 COMMIT} msg] $msg } {1 {database is locked}} + sql1 ROLLBACK + + # In this case, because the "SELECT * FROM t1" is first stepped before + # the "BEGIN CONCURRENT", the pages it reads are not recorded by the + # pager object. And so the transaction can be committed. Technically + # this behaviour (the effect of an ongoing SELECT on a BEGIN CONCURRENT + # transacation) is undefined. + # + do_test 10.$tn.2.1 { + code1 { + set ::stmt [sqlite3_prepare db "SELECT * FROM t1" -1 dummy] + sqlite3_step $::stmt + } + } {SQLITE_ROW} + do_test 10.$tn.2.2 { + sql1 { + BEGIN CONCURRENT; + INSERT INTO t2 VALUES(4); + } + code1 { + set res [list] + lappend res [sqlite3_column_int $::stmt 0] + while {[sqlite3_step $::stmt]=="SQLITE_ROW"} { + lappend res [sqlite3_column_int $::stmt 0] + } + sqlite3_finalize $::stmt + set res + } + } {1 2 3 4} + do_test 10.$tn.2.3 { + sql2 { INSERT INTO t1 VALUES(5) } + sql1 COMMIT + } {} + + # More tests surrounding long-lived prepared statements and concurrent + # transactions. + do_test 10.$tn.3.1 { + sql1 { + BEGIN CONCURRENT; + SELECT * FROM t1; + COMMIT; + } + sql1 { + BEGIN CONCURRENT; + INSERT INTO t2 VALUES(5); + } + sql2 { + INSERT INTO t1 VALUES(5); + } + sql1 COMMIT + sql3 { + SELECT * FROM t2; + } + } {1 2 3 4 5} + do_test 10.$tn.3.2 { + sql1 { + BEGIN CONCURRENT; + SELECT * FROM t1; + ROLLBACK; + } + sql1 { + BEGIN CONCURRENT; + INSERT INTO t2 VALUES(6); + } + sql2 { + INSERT INTO t1 VALUES(6); + } + sql1 COMMIT + sql3 { SELECT * FROM t2 } + } {1 2 3 4 5 6} + do_test 10.$tn.3.3 { + sql1 { BEGIN CONCURRENT } + code1 { + set ::stmt [sqlite3_prepare db "SELECT * FROM t1" -1 dummy] + sqlite3_step $::stmt + } + sql1 { + INSERT INTO t2 VALUES(7); + SELECT * FROM t3; + ROLLBACK; + BEGIN CONCURRENT; + } + sql2 { INSERT INTO t3 VALUES(5) } + code1 { sqlite3_finalize $::stmt } + sql1 { + INSERT INTO t2 VALUES(8); + COMMIT; + } + } {} } From c8a9d15887ea35beaaa6357bb73fac24cd91a679 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 27 Aug 2015 19:57:16 +0000 Subject: [PATCH 027/179] Add header comments for new methods in pager.c. FossilOrigin-Name: 437c7e219d3240767a28f73487bc26c3be3044b3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 13 +++++++++++++ 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index a9bcad179f..605728fd86 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\scases\sfor\sconcurrent\stransactions\sand\slong-lived\sSELECT\sstatements. -D 2015-08-27T19:22:56.303 +C Add\sheader\scomments\sfor\snew\smethods\sin\spager.c. +D 2015-08-27T19:57:16.810 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -324,7 +324,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 388c023582b17890f10c980b30ec1922b471753b F src/os_win.c 40b3af7a47eb1107d0d69e592bec345a3b7b798a F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 732f3b107ac7180e4c628114a5d06dda00393080 +F src/pager.c 4343baff2f7e35f11988deb9c213870432ded1b1 F src/pager.h f00930ca3bfc0f3298b08a69ed7b920e89b531de F src/parse.y 1e645cacb93979c59f2a510ee2c100e769bd5e3c F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0 @@ -1382,7 +1382,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fc17f73170a27c2fe511ed6b6d488535c4e35bae -R 54233c250c37b5d4c5cad57267aa2abd +P fd4798cb7af263409c20d3cf81236b830bd68570 +R f446f1e219f1b69d13fe21a44df99a43 U dan -Z 1ef37227dc1b9c5caa6e609be7886347 +Z 0a2b89af585f5781988dd453201696be diff --git a/manifest.uuid b/manifest.uuid index 55734e47ba..da0a46abb3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fd4798cb7af263409c20d3cf81236b830bd68570 \ No newline at end of file +437c7e219d3240767a28f73487bc26c3be3044b3 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 7db7544552..2b20603379 100644 --- a/src/pager.c +++ b/src/pager.c @@ -1742,6 +1742,12 @@ static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ } #ifdef SQLITE_ENABLE_CONCURRENT +/* +** If they are not already, begin recording all pages read from the pager layer +** by the b-tree layer This is used by concurrent transactions. Return +** SQLITE_OK if successful, or an SQLite error code (SQLITE_NOMEM) if an error +** occurs. +*/ int sqlite3PagerBeginConcurrent(Pager *pPager){ int rc = SQLITE_OK; if( pPager->pAllRead==0 ){ @@ -1753,11 +1759,18 @@ int sqlite3PagerBeginConcurrent(Pager *pPager){ return rc; } +/* +** Stop recording all pages read from the pager layer by the b-tree layer +** and discard any current records. +*/ void sqlite3PagerEndConcurrent(Pager *pPager){ sqlite3BitvecDestroy(pPager->pAllRead); pPager->pAllRead = 0; } +/* +** Return true if the database is in wal mode. False otherwise. +*/ int sqlite3PagerIsWal(Pager *pPager){ return pPager->pWal!=0; } From 3f531da5648e7f774d44562cfb855f0322423004 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 1 Sep 2015 17:48:54 +0000 Subject: [PATCH 028/179] Fixes so that it builds without warnings both with and without SQLITE_ENABLE_CONCURRENT. FossilOrigin-Name: 5ed2a445a164f0f0c2669c6681ea76618e639961 --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/btree.c | 4 +++- src/pager.c | 2 ++ src/pager.h | 1 + src/wal.c | 2 +- src/wal.h | 6 ++++++ 7 files changed, 25 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 98f9a5d524..9d924de776 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\swith\sthis\sbranch. -D 2015-08-28T09:27:51.910 +C Fixes\sso\sthat\sit\sbuilds\swithout\swarnings\sboth\swith\sand\swithout\nSQLITE_ENABLE_CONCURRENT. +D 2015-09-01T17:48:54.487 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -279,7 +279,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 4d9134dc988a87838c06056c89c0e8c4700a0452 F src/bitvec.c d1f21d7d91690747881f03940584f4cc548c9d3d F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 -F src/btree.c 88ff0f5bdcd200e4b8cdc2f49a89369112a592d7 +F src/btree.c ca6c7e4bbbf91b6c91d0278942b917e81d3fc18d F src/btree.h 00d4cdb747c4172a5566faf037116985dbbc377e F src/btreeInt.h 171864bcd81635583dab7b8a04b19b454b18ef80 F src/build.c bb9335e06a6f308c595e0507ce97daf791735a6e @@ -324,8 +324,8 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 388c023582b17890f10c980b30ec1922b471753b F src/os_win.c 40b3af7a47eb1107d0d69e592bec345a3b7b798a F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 4343baff2f7e35f11988deb9c213870432ded1b1 -F src/pager.h f00930ca3bfc0f3298b08a69ed7b920e89b531de +F src/pager.c 284edd20ae51f1f52a539685ce42c7a1e61ac716 +F src/pager.h 174fd2cd7091c87418616c62582a5c00cb8da2ac F src/parse.y 1e645cacb93979c59f2a510ee2c100e769bd5e3c F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0 F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9 @@ -411,8 +411,8 @@ F src/vdbesort.c f5009e7a35e3065635d8918b9a31f498a499976b F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 F src/vtab.c d31174e4c8f592febab3fa7f69e18320b4fd657a F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb -F src/wal.c 44ec009f7742a660d3558845e47e870af4542689 -F src/wal.h 903ef67e17f8b466dc7cfc4186fc23e80be10ff8 +F src/wal.c d632ee33340554888a7848a253a580167a031cdb +F src/wal.h ec78c303882bc04b3d0b86f0ddbc87309401595a F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba F src/where.c 91e73ffc699c140a59baa03a6b7b060db02bed81 F src/whereInt.h 901c17c1e3c82745ad9b85b4471543fa59c980e9 @@ -1383,7 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 437c7e219d3240767a28f73487bc26c3be3044b3 64abb65d4df11e5b3bcc4afc8e7c18e907c6080a -R d1f878aab8d7c75cbf15e5d6b4c78c00 -U dan -Z 64a6a9bedfd1c579ad90113faf74b29f +P 57bc0194f41dbcd2c343e665e7af475cd4dd7328 +R 3e87a7400b2863c0ecf368784d9e77cc +U drh +Z 6775a99654ddab524e15a28b9025b05e diff --git a/manifest.uuid b/manifest.uuid index dabc379273..407c9dbaa4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -57bc0194f41dbcd2c343e665e7af475cd4dd7328 \ No newline at end of file +5ed2a445a164f0f0c2669c6681ea76618e639961 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 9986c95325..d2b6cdd4ab 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4009,7 +4009,9 @@ static int btreeFixUnlocked(Btree *p){ return rc; } -#endif +#else +# define btreeFixUnlocked(X) SQLITE_OK +#endif /* ENABLE_CONCURRENT */ /* ** This routine does the first phase of a two-phase commit. This routine diff --git a/src/pager.c b/src/pager.c index 2b20603379..5b1788ef4f 100644 --- a/src/pager.c +++ b/src/pager.c @@ -906,7 +906,9 @@ static int assert_pager_state(Pager *p){ if( !pagerUseWal(pPager) ){ assert( p->eLock>=RESERVED_LOCK ); } +#ifdef SQLITE_ENABLE_CONCURRENT assert( pPager->dbSize==pPager->dbOrigSize || pPager->pAllRead ); +#endif assert( pPager->dbOrigSize==pPager->dbFileSize ); assert( pPager->dbOrigSize==pPager->dbHintSize ); assert( pPager->setMaster==0 ); diff --git a/src/pager.h b/src/pager.h index 1df91bbca0..538a611cf9 100644 --- a/src/pager.h +++ b/src/pager.h @@ -200,6 +200,7 @@ int sqlite3PagerBeginConcurrent(Pager*); void sqlite3PagerDropExclusiveLock(Pager*); int sqlite3PagerUpgradeSnapshot(Pager *pPager, DbPage*); void sqlite3PagerSetDbsize(Pager *pPager, Pgno); +int sqlite3PagerIsWal(Pager*); #else # define sqlite3PagerEndConcurrent(x) #endif diff --git a/src/wal.c b/src/wal.c index 445d084628..fbc92238bb 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2489,7 +2489,7 @@ int sqlite3WalFindFrame( #endif *piRead = iRead; - return SQLITE_OK; + return rc; } /* diff --git a/src/wal.h b/src/wal.h index 35ffe4d38c..4d9e1b0816 100644 --- a/src/wal.h +++ b/src/wal.h @@ -126,8 +126,14 @@ int sqlite3WalExclusiveMode(Wal *pWal, int op); */ int sqlite3WalHeapMemory(Wal *pWal); +#ifdef SQLITE_ENABLE_CONCURRENT +/* Tell the wal layer that we want to commit a concurrent transaction */ int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPg, Bitvec *pRead); + +/* Upgrade the state of the client to take into account changes written +** by other connections */ int sqlite3WalUpgradeSnapshot(Wal *pWal); +#endif #ifdef SQLITE_ENABLE_ZIPVFS /* If the WAL file is not empty, return the number of bytes of content From 01be463eeb60f3bf29c0d4f7819aeb366fd08645 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 3 Sep 2015 15:17:12 +0000 Subject: [PATCH 029/179] Changes from ENABLE_CONCURRENT (default off) to OMIT_CONCURRENT (default on). This is not a clear-cut decision and might be changed back. FossilOrigin-Name: f8ae9bfd05abc35293ad6bc62ab1bdbe357d964e --- manifest | 28 ++++++++++++++-------------- manifest.uuid | 2 +- src/btree.c | 32 +++++++++++++++++++------------- src/btreeInt.h | 14 +++++++------- src/pager.c | 36 ++++++++++++++++++++---------------- src/pager.h | 2 +- src/test_config.c | 2 +- src/vdbe.c | 4 ++-- src/vdbeaux.c | 2 +- src/wal.c | 7 ++++--- src/wal.h | 4 ++-- 11 files changed, 72 insertions(+), 61 deletions(-) diff --git a/manifest b/manifest index 0ab2bd2e03..5e033198ad 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\soptimizations. -D 2015-09-03T14:04:05.544 +C Changes\sfrom\sENABLE_CONCURRENT\s(default\soff)\sto\sOMIT_CONCURRENT\s(default\son).\nThis\sis\snot\sa\sclear-cut\sdecision\sand\smight\sbe\schanged\sback. +D 2015-09-03T15:17:12.457 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -279,9 +279,9 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 4d9134dc988a87838c06056c89c0e8c4700a0452 F src/bitvec.c d1f21d7d91690747881f03940584f4cc548c9d3d F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 -F src/btree.c 89f97b912a959d82c1ab399da3057c93505cfa8f +F src/btree.c 7663d05467277379fd29ae44eb82bcbdee7f618f F src/btree.h 00d4cdb747c4172a5566faf037116985dbbc377e -F src/btreeInt.h 171864bcd81635583dab7b8a04b19b454b18ef80 +F src/btreeInt.h df0e92901c6fbb01aa8fab3cfbcdaaba2654fd04 F src/build.c 565d84cf1d80e054e15907641985584bbd5eadf6 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f @@ -324,8 +324,8 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 76f493ed71c4154338049dee1bf6e47f69c74a55 F src/os_win.c 40b3af7a47eb1107d0d69e592bec345a3b7b798a F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 284edd20ae51f1f52a539685ce42c7a1e61ac716 -F src/pager.h 174fd2cd7091c87418616c62582a5c00cb8da2ac +F src/pager.c 567093b5d35cba6e5a4316e1611f6bd44415c0d7 +F src/pager.h 244606cccd4293a2bc4be20bb37bf730028c5c18 F src/parse.y 1e645cacb93979c59f2a510ee2c100e769bd5e3c F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0 F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9 @@ -361,7 +361,7 @@ F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12 F src/test_backup.c 2e6e6a081870150f20c526a2e9d0d29cda47d803 F src/test_blob.c e5a7a81d61a780da79101aeb1e60d300af169e07 F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f -F src/test_config.c f853203a1d2035370ea26ad48248f6d00a10b752 +F src/test_config.c 24f1c8415e57d709ae603cbde42a93990192c8fd F src/test_demovfs.c 0de72c2c89551629f58486fde5734b7d90758852 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f @@ -400,19 +400,19 @@ F src/update.c 795fba8ebadeb194285b0a73a07f7ad5ae4d0410 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c fc612367108b74573c5fd13a85d0a23027f438bd F src/vacuum.c 983cc3754718ef169a6ea9aef86798bd28106f21 -F src/vdbe.c 791a28761cf6176d11ec3e41cdf8e749c65384ff +F src/vdbe.c 8e1d159743af4b17fce88037a46d68b280aaa729 F src/vdbe.h 4bc88bd0e06f8046ee6ab7487c0015e85ad949ad F src/vdbeInt.h 8b54e01ad0463590e7cffabce0bc36da9ee4f816 F src/vdbeapi.c bda74ef4b5103d7b4a4be36f936d3cf2b56a7d6f -F src/vdbeaux.c 33e2973796a016956887466c2b3a1991eab3af6c +F src/vdbeaux.c 4988b83d1e1989ee554b2fa4ca18f3606a78437c F src/vdbeblob.c 4f2e8e075d238392df98c5e03a64342465b03f90 F src/vdbemem.c ae38a0d35ae71cf604381a887c170466ba518090 F src/vdbesort.c f5009e7a35e3065635d8918b9a31f498a499976b F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 F src/vtab.c 2ecfe020c10e0a0c7b078203fdba2fae844744bc F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb -F src/wal.c 00352cb9d23c5599f219496d4b22f30d4e30f8ac -F src/wal.h ec78c303882bc04b3d0b86f0ddbc87309401595a +F src/wal.c 5a86298540935981eea840050f66e516dbe536af +F src/wal.h 361b16891d2772294b138054c84f5a3bad6e9d05 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba F src/where.c f0e08e4f1f66ba1a0f5b70c5161cb031ce1fb858 F src/whereInt.h 901c17c1e3c82745ad9b85b4471543fa59c980e9 @@ -1383,7 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3dea047465fa8e3031046a33016b6ed760336ddc 847387ec8e6fef283899578fb232b2c23b00ee5b -R 7c90873b273aeaf2eed231afaea97e29 +P 71e7299e8d501618b10f8c1f78572789b1b6853a +R cf8206aeb5613d1354992f43b08834e8 U drh -Z 99e28d5734be346b4e9234313f30bb5e +Z 1d56cb0a736729e62628d2ca4ac46f41 diff --git a/manifest.uuid b/manifest.uuid index 3c32784a23..1701931440 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -71e7299e8d501618b10f8c1f78572789b1b6853a \ No newline at end of file +f8ae9bfd05abc35293ad6bc62ab1bdbe357d964e \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 99051e40c4..0c6ccd403a 100644 --- a/src/btree.c +++ b/src/btree.c @@ -440,7 +440,7 @@ static void downgradeAllSharedCacheTableLocks(Btree *p){ #endif /* SQLITE_OMIT_SHARED_CACHE */ -#ifdef SQLITE_ENABLE_CONCURRENT +#ifndef SQLITE_OMIT_CONCURRENT /* ** The following structure - BtreePtrmap - stores the in-memory pointer map ** used for newly allocated pages in CONCURRENT transactions. Such pages are @@ -473,7 +473,8 @@ struct BtreePtrmap { RollbackEntry *aRollback; /* Array of rollback entries */ }; -/* +/* !defined(SQLITE_OMIT_CONCURRENT) +** ** If page number pgno is greater than or equal to BtreePtrmap.iFirst, ** store an entry for it in the pointer-map structure. */ @@ -531,7 +532,8 @@ static int btreePtrmapStore( return SQLITE_OK; } -/* +/* !defined(SQLITE_OMIT_CONCURRENT) +** ** Open savepoint iSavepoint, if it is not already open. */ static int btreePtrmapBegin(BtShared *pBt, int nSvpt){ @@ -558,7 +560,8 @@ static int btreePtrmapBegin(BtShared *pBt, int nSvpt){ return SQLITE_OK; } -/* +/* !defined(SQLITE_OMIT_CONCURRENT) +** ** Rollback (if op==SAVEPOINT_ROLLBACK) or release (if op==SAVEPOINT_RELEASE) ** savepoint iSvpt. */ @@ -587,7 +590,8 @@ static void btreePtrmapEnd(BtShared *pBt, int op, int iSvpt){ } } -/* +/* !defined(SQLITE_OMIT_CONCURRENT) +** ** This function is called after an CONCURRENT transaction is opened on the ** database. It allocates the BtreePtrmap structure used to track pointers ** to allocated pages and zeroes the nFree/iTrunk fields in the database @@ -609,7 +613,8 @@ static int btreePtrmapAllocate(BtShared *pBt){ return rc; } -/* +/* !defined(SQLITE_OMIT_CONCURRENT) +** ** Free any BtreePtrmap structure allocated by an earlier call to ** btreePtrmapAllocate(). */ @@ -623,12 +628,12 @@ static void btreePtrmapDelete(BtShared *pBt){ pBt->pMap = 0; } } -#else +#else /* SQLITE_OMIT_CONCURRENT */ # define btreePtrmapAllocate(x) SQLITE_OK # define btreePtrmapDelete(x) # define btreePtrmapBegin(x,y) SQLITE_OK # define btreePtrmapEnd(x,y,z) -#endif +#endif /* SQLITE_OMIT_CONCURRENT */ static void releasePage(MemPage *pPage); /* Forward reference */ @@ -1075,7 +1080,7 @@ static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){ /* The master-journal page number is never added to a pointer-map page */ assert( 0==PTRMAP_ISPAGE(pBt, PENDING_BYTE_PAGE(pBt)) ); -#ifdef SQLITE_ENABLE_CONCURRENT +#ifndef SQLITE_OMIT_CONCURRENT if( pBt->pMap ){ *pRC = btreePtrmapStore(pBt, key, eType, parent); return; @@ -3397,7 +3402,7 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ trans_begun: -#ifdef SQLITE_ENABLE_CONCURRENT +#ifndef SQLITE_OMIT_CONCURRENT if( bConcurrent && rc==SQLITE_OK && sqlite3PagerIsWal(pBt->pPager) ){ rc = sqlite3PagerBeginConcurrent(pBt->pPager); if( rc==SQLITE_OK && wrflag ){ @@ -3852,7 +3857,7 @@ static int autoVacuumCommit(BtShared *pBt){ # define setChildPtrmaps(x) SQLITE_OK #endif -#ifdef SQLITE_ENABLE_CONCURRENT +#ifndef SQLITE_OMIT_CONCURRENT /* ** This function is called as part of merging an CONCURRENT transaction with ** the snapshot at the head of the wal file. It relocates all pages in the @@ -3920,7 +3925,8 @@ static int btreeRelocateRange( return rc; } -/* +/* !defined(SQLITE_OMIT_CONCURRENT) +** ** The b-tree handle passed as the only argument is about to commit an ** CONCURRENT transaction. At this point it is guaranteed that this is ** possible - the wal WRITER lock is held and it is known that there are @@ -4011,7 +4017,7 @@ static int btreeFixUnlocked(Btree *p){ } #else # define btreeFixUnlocked(X) SQLITE_OK -#endif /* ENABLE_CONCURRENT */ +#endif /* SQLITE_OMIT_CONCURRENT */ /* ** This routine does the first phase of a two-phase commit. This routine diff --git a/src/btreeInt.h b/src/btreeInt.h index e686466e81..e1af63f0bb 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -448,7 +448,7 @@ struct BtShared { Btree *pWriter; /* Btree with currently open write transaction */ #endif u8 *pTmpSpace; /* Temp space sufficient to hold a single cell */ -#ifdef SQLITE_ENABLE_CONCURRENT +#ifndef SQLITE_OMIT_CONCURRENT BtreePtrmap *pMap; #endif }; @@ -657,16 +657,16 @@ struct BtCursor { ** (sqliteMallocRaw), it is not possible to use conditional compilation. ** So, this macro is defined instead. */ -#ifndef SQLITE_OMIT_AUTOVACUUM -#define ISAUTOVACUUM (pBt->autoVacuum) -#else +#ifdef SQLITE_OMIT_AUTOVACUUM #define ISAUTOVACUUM 0 +#else +#define ISAUTOVACUUM (pBt->autoVacuum) #endif -#ifdef SQLITE_ENABLE_CONCURRENT -# define ISCONCURRENT (pBt->pMap!=0) -#else +#ifdef SQLITE_OMIT_CONCURRENT # define ISCONCURRENT 0 +#else +# define ISCONCURRENT (pBt->pMap!=0) #endif #define REQUIRE_PTRMAP (ISAUTOVACUUM || ISCONCURRENT) diff --git a/src/pager.c b/src/pager.c index 5b1788ef4f..c72069bbf6 100644 --- a/src/pager.c +++ b/src/pager.c @@ -657,7 +657,7 @@ struct Pager { u32 cksumInit; /* Quasi-random value added to every checksum */ u32 nSubRec; /* Number of records written to sub-journal */ Bitvec *pInJournal; /* One bit for each page in the database file */ -#ifdef SQLITE_ENABLE_CONCURRENT +#ifndef SQLITE_OMIT_CONCURRENT Bitvec *pAllRead; /* Pages read within current CONCURRENT trans. */ #endif sqlite3_file *fd; /* File descriptor for database */ @@ -906,7 +906,7 @@ static int assert_pager_state(Pager *p){ if( !pagerUseWal(pPager) ){ assert( p->eLock>=RESERVED_LOCK ); } -#ifdef SQLITE_ENABLE_CONCURRENT +#ifndef SQLITE_OMIT_CONCURRENT assert( pPager->dbSize==pPager->dbOrigSize || pPager->pAllRead ); #endif assert( pPager->dbOrigSize==pPager->dbFileSize ); @@ -1743,7 +1743,7 @@ static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ return rc; } -#ifdef SQLITE_ENABLE_CONCURRENT +#ifndef SQLITE_OMIT_CONCURRENT /* ** If they are not already, begin recording all pages read from the pager layer ** by the b-tree layer This is used by concurrent transactions. Return @@ -1761,7 +1761,8 @@ int sqlite3PagerBeginConcurrent(Pager *pPager){ return rc; } -/* +/* !defined(SQLITE_OMIT_CONCURRENT) +** ** Stop recording all pages read from the pager layer by the b-tree layer ** and discard any current records. */ @@ -1770,13 +1771,14 @@ void sqlite3PagerEndConcurrent(Pager *pPager){ pPager->pAllRead = 0; } -/* +/* !defined(SQLITE_OMIT_CONCURRENT) +** ** Return true if the database is in wal mode. False otherwise. */ int sqlite3PagerIsWal(Pager *pPager){ return pPager->pWal!=0; } -#endif +#endif /* SQLITE_OMIT_CONCURRENT */ /* ** Free the Pager.pInJournal and Pager.pAllRead bitvec objects. @@ -3069,10 +3071,10 @@ static int pagerRollbackWal(Pager *pPager){ rc = sqlite3WalUndo(pPager->pWal, pagerUndoCallback, (void *)pPager); pList = sqlite3PcacheDirtyList(pPager->pPCache); +#ifndef SQLITE_OMIT_CONCURRENT /* If this is an CONCURRENT transaction, then page 1 must be reread from ** the db file, even if it is not dirty. This is because the b-tree layer ** may have already zeroed the nFree and iTrunk header fields. */ -#ifdef SQLITE_ENABLE_CONCURRENT if( rc==SQLITE_OK && (pList==0 || pList->pgno!=1) && pPager->pAllRead ){ rc = pagerUndoCallback((void*)pPager, 1); } @@ -4486,9 +4488,9 @@ static int pagerStress(void *p, PgHdr *pPg){ pPg->pDirty = 0; if( pagerUseWal(pPager) ){ +#ifndef SQLITE_OMIT_CONCURRENT /* If the transaction is a "BEGIN CONCURRENT" transaction, the page ** cannot be flushed to disk. Return early in this case. */ -#ifdef SQLITE_ENABLE_CONCURRENT if( pPager->pAllRead ) return SQLITE_OK; #endif @@ -5332,10 +5334,10 @@ int sqlite3PagerAcquire( } pPager->hasBeenUsed = 1; +#ifndef SQLITE_OMIT_CONCURRENT /* If this is an CONCURRENT transaction and the page being read was ** present in the database file when the transaction was opened, ** mark it as read in the pAllRead vector. */ -#ifdef SQLITE_ENABLE_CONCURRENT if( pPager->pAllRead && pgno<=pPager->dbOrigSize ){ rc = sqlite3BitvecSet(pPager->pAllRead, pgno); if( rc!=SQLITE_OK ) goto pager_acquire_err; @@ -5970,7 +5972,7 @@ int sqlite3PagerWrite(PgHdr *pPg){ ** to sqlite3PagerWrite(). In other words, return TRUE if it is ok ** to change the content of the page. */ -#if defined(SQLITE_ENABLE_CONCURRENT) || !defined(NDEBUG) +#if !defined(SQLITE_OMIT_CONCURRENT) || !defined(NDEBUG) int sqlite3PagerIswriteable(DbPage *pPg){ return pPg->flags & PGHDR_WRITEABLE; } @@ -6151,7 +6153,7 @@ int sqlite3PagerExclusiveLock(Pager *pPager, PgHdr *pPage1){ if( 0==pagerUseWal(pPager) ){ rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); } -#ifdef SQLITE_ENABLE_CONCURRENT +#ifndef SQLITE_OMIT_CONCURRENT else{ if( pPager->pAllRead ){ /* This is an CONCURRENT transaction. Attempt to lock the wal database @@ -6165,11 +6167,11 @@ int sqlite3PagerExclusiveLock(Pager *pPager, PgHdr *pPage1){ ); } } -#endif +#endif /* SQLITE_OMIT_CONCURRENT */ return rc; } -#ifdef SQLITE_ENABLE_CONCURRENT +#ifndef SQLITE_OMIT_CONCURRENT /* ** This function is called as part of committing an CONCURRENT transaction. ** At this point the wal WRITER lock is held, and all pages in the cache @@ -6198,14 +6200,16 @@ int sqlite3PagerUpgradeSnapshot(Pager *pPager, DbPage *pPage1){ return rc; } -/* +/* !defined(SQLITE_OMIT_CONCURRENT) +** ** Set the in-memory cache of the database file size to nSz pages. */ void sqlite3PagerSetDbsize(Pager *pPager, Pgno nSz){ pPager->dbSize = nSz; } -/* +/* !defined(SQLITE_OMIT_CONCURRENT) +** ** If this is a WAL mode connection and the WRITER lock is currently held, ** relinquish it. */ @@ -6214,7 +6218,7 @@ void sqlite3PagerDropExclusiveLock(Pager *pPager){ sqlite3WalEndWriteTransaction(pPager->pWal); } } -#endif /* ifdef SQLITE_ENABLE_CONCURRENT */ +#endif /* SQLITE_OMIT_CONCURRENT */ /* diff --git a/src/pager.h b/src/pager.h index 538a611cf9..510ce85b3d 100644 --- a/src/pager.h +++ b/src/pager.h @@ -194,7 +194,7 @@ void sqlite3PagerTruncateImage(Pager*,Pgno); void sqlite3PagerRekey(DbPage*, Pgno, u16); -#ifdef SQLITE_ENABLE_CONCURRENT +#ifndef SQLITE_OMIT_CONCURRENT void sqlite3PagerEndConcurrent(Pager*); int sqlite3PagerBeginConcurrent(Pager*); void sqlite3PagerDropExclusiveLock(Pager*); diff --git a/src/test_config.c b/src/test_config.c index d153697715..d9752ae237 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -573,7 +573,7 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY); Tcl_SetVar2(interp, "sqlite_options", "truncate_opt", "1", TCL_GLOBAL_ONLY); #endif -#ifdef SQLITE_ENABLE_CONCURRENT +#ifndef SQLITE_OMIT_CONCURRENT Tcl_SetVar2(interp, "sqlite_options", "concurrent", "1", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "concurrent", "0", TCL_GLOBAL_ONLY); diff --git a/src/vdbe.c b/src/vdbe.c index bf14bd1173..e07c89779a 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3227,7 +3227,7 @@ case OP_SetCookie: { /* in3 */ assert( sqlite3SchemaMutexHeld(db, pOp->p1, 0) ); pIn3 = &aMem[pOp->p3]; sqlite3VdbeMemIntegerify(pIn3); -#ifdef SQLITE_ENABLE_CONCURRENT +#ifndef SQLITE_OMIT_CONCURRENT if( db->bConcurrent && (pOp->p2==BTREE_USER_VERSION || pOp->p2==BTREE_APPLICATION_ID) ){ @@ -6115,7 +6115,7 @@ case OP_Expire: { */ case OP_TableLock: { u8 isWriteLock = (u8)pOp->p3; -#ifdef SQLITE_ENABLE_CONCURRENT +#ifndef SQLITE_OMIT_CONCURRENT if( isWriteLock && db->bConcurrent && pOp->p2==1 ){ rc = SQLITE_ERROR; sqlite3VdbeError(p, diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 61f0774878..e085f313c0 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2041,7 +2041,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ } } -#ifdef SQLITE_ENABLE_CONCURRENT +#ifndef SQLITE_OMIT_CONCURRENT if( db->bConcurrent && (rc & 0xFF)==SQLITE_BUSY ){ /* An SQLITE_BUSY or SQLITE_BUSY_SNAPSHOT was encountered while ** attempting to take the WRITER lock on a wal file. Release the diff --git a/src/wal.c b/src/wal.c index 07f06293ac..23281740ed 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2626,7 +2626,7 @@ static int walUpgradeReadlock(Wal *pWal){ } -#ifdef SQLITE_ENABLE_CONCURRENT +#ifndef SQLITE_OMIT_CONCURRENT /* ** This function is only ever called when committing a "BEGIN CONCURRENT" ** transaction. It may be assumed that no frames have been written to @@ -2752,7 +2752,8 @@ int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPage1, Bitvec *pAllRead){ return rc; } -/* +/* !defined(SQLITE_OMIT_CONCURRENT) +** ** This function is called as part of committing an CONCURRENT transaction. ** It is assumed that sqlite3WalLockForCommit() has already been successfully ** called and so (a) the WRITER lock is held and (b) it is known that the @@ -2778,7 +2779,7 @@ int sqlite3WalUpgradeSnapshot(Wal *pWal){ } return rc; } -#endif /* SQLITE_ENABLE_CONCURRENT */ +#endif /* SQLITE_OMIT_CONCURRENT */ /* ** End a write transaction. The commit has already been done. This diff --git a/src/wal.h b/src/wal.h index 4d9e1b0816..db6426392a 100644 --- a/src/wal.h +++ b/src/wal.h @@ -126,14 +126,14 @@ int sqlite3WalExclusiveMode(Wal *pWal, int op); */ int sqlite3WalHeapMemory(Wal *pWal); -#ifdef SQLITE_ENABLE_CONCURRENT +#ifndef SQLITE_OMIT_CONCURRENT /* Tell the wal layer that we want to commit a concurrent transaction */ int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPg, Bitvec *pRead); /* Upgrade the state of the client to take into account changes written ** by other connections */ int sqlite3WalUpgradeSnapshot(Wal *pWal); -#endif +#endif /* SQLITE_OMIT_CONCURRENT */ #ifdef SQLITE_ENABLE_ZIPVFS /* If the WAL file is not empty, return the number of bytes of content From 144cd3df05e069c256d05553dd39da505353ede2 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 4 Sep 2015 16:39:16 +0000 Subject: [PATCH 030/179] Remove the EXCLUSIVE and CONCURRENT tokens from the tokenizer. Let the BEGIN statement be followed by an ID, but throw a syntax error if the ID is anything other than EXCLUSIVE or CONCURRENT. FossilOrigin-Name: c0bf92eca4d3d4666e8a0476ef30fa8123de1cb0 --- addopcodes.awk | 2 ++ manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/parse.y | 32 ++++++++++++++++++++++++-------- tool/mkkeywordhash.c | 2 -- 5 files changed, 35 insertions(+), 19 deletions(-) diff --git a/addopcodes.awk b/addopcodes.awk index dcd31eff84..c5be40b74b 100644 --- a/addopcodes.awk +++ b/addopcodes.awk @@ -31,4 +31,6 @@ END { printf "#define TK_%-29s %4d\n", "UMINUS", ++max printf "#define TK_%-29s %4d\n", "UPLUS", ++max printf "#define TK_%-29s %4d\n", "REGISTER", ++max + printf "#define TK_%-29s %4d\n", "EXCLUSIVE", ++max + printf "#define TK_%-29s %4d\n", "CONCURRENT", ++max } diff --git a/manifest b/manifest index 2eb676f458..d8ae9b24e4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sperformance\senhancements\sfrom\strunk.\s\sThis\sbranch\snow\sruns\s(slightly)\nfaster\sthan\sthe\s3.8.11.1\srelease,\sthough\sstill\sslightly\sslower\sthan\strunk. -D 2015-09-03T20:52:44.863 +C Remove\sthe\sEXCLUSIVE\sand\sCONCURRENT\stokens\sfrom\sthe\stokenizer.\s\sLet\sthe\nBEGIN\sstatement\sbe\sfollowed\sby\san\sID,\sbut\sthrow\sa\ssyntax\serror\sif\sthe\sID\nis\sanything\sother\sthan\sEXCLUSIVE\sor\sCONCURRENT. +D 2015-09-04T16:39:16.988 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -8,7 +8,7 @@ F Makefile.vxworks e1b65dea203f054e71653415bd8f96dcaed47858 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 F VERSION ccfc4d1576dbfdeece0a4372a2e6a2e37d3e7975 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 -F addopcodes.awk 9eb448a552d5c0185cf62c463f9c173cedae3811 +F addopcodes.awk 694f522754f0a32dc53bc73ccebf02f28ab10a54 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 @@ -326,7 +326,7 @@ F src/os_win.c 40b3af7a47eb1107d0d69e592bec345a3b7b798a F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca F src/pager.c 06e10a0b736ba22df52e0f56fa67a42de41a0445 F src/pager.h 244606cccd4293a2bc4be20bb37bf730028c5c18 -F src/parse.y 1e645cacb93979c59f2a510ee2c100e769bd5e3c +F src/parse.y 0dcce0ae4e106cdaad4053f73b3bc2dedca9c6d6 F src/pcache.c 24be750c79272e0ca7b6e007bc94999700f3e5ef F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9 F src/pcache1.c e1529369c047ac645e6a28196f25b7e936c46b82 @@ -1343,7 +1343,7 @@ F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh d1a2da0e15b2ed33d60af35c7e9d483f13a8eb9f -F tool/mkkeywordhash.c 8d78ea188240bc08ec080adf3af84717f013e69a +F tool/mkkeywordhash.c ff6fc41e733204e289b38b9cd3808701725c6793 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkpragmatab.tcl 84af2b180484323a2ea22a2279e8bd9e3e1e492e F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 @@ -1383,7 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f8ae9bfd05abc35293ad6bc62ab1bdbe357d964e f00a9e1e998c4bd249a45444dc2d71a7e4903b8b -R 26478d9405c36063665d5a8b96df78bb +P c490bfb150c66763226a35e30ba289abbf29906d +R 1a77cff5bf225c8f1af4050df926d2bd U drh -Z 60e0d06fc5e4f945ca55d9cbf383722d +Z 4bd187d1d0f74fd2a4074d09ae209a9a diff --git a/manifest.uuid b/manifest.uuid index bf8f7c634e..9cb3d0d03e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c490bfb150c66763226a35e30ba289abbf29906d \ No newline at end of file +c0bf92eca4d3d4666e8a0476ef30fa8123de1cb0 \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index 7a05ecab6a..63bf5e4715 100644 --- a/src/parse.y +++ b/src/parse.y @@ -32,7 +32,7 @@ %syntax_error { UNUSED_PARAMETER(yymajor); /* Silence some compiler warnings */ assert( TOKEN.z[0] ); /* The tokenizer always gives us a token */ - sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN); + parserSyntaxError(pParse, &TOKEN); } %stack_overflow { UNUSED_PARAMETER(yypMinor); /* Silence some compiler warnings */ @@ -94,6 +94,13 @@ struct TrigEvent { int a; IdList * b; }; */ struct AttachKey { int type; Token key; }; +/* +** Generate a syntax error +*/ +static void parserSyntaxError(Parse *pParse, Token *p){ + sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", p); +} + } // end %include // Input is a single SQL command @@ -118,10 +125,18 @@ trans_opt ::= TRANSACTION. trans_opt ::= TRANSACTION nm. %type transtype {int} transtype(A) ::= . {A = TK_DEFERRED;} -transtype(A) ::= DEFERRED(X). {A = @X;} -transtype(A) ::= IMMEDIATE(X). {A = @X;} -transtype(A) ::= EXCLUSIVE(X). {A = @X;} -transtype(A) ::= CONCURRENT(X). {A = @X;} +transtype(A) ::= DEFERRED(X). {A = @X;} +transtype(A) ::= IMMEDIATE(X). {A = @X;} +transtype(A) ::= ID(X). { + Token *p = &X; + if( p->n==9 && sqlite3_strnicmp(p->z,"exclusive",9)==0 ){ + A = TK_EXCLUSIVE; + }else if( p->n==10 && sqlite3_strnicmp(p->z,"concurrent",10)==0 ){ + A = TK_CONCURRENT; + }else{ + parserSyntaxError(pParse, p); + } +} cmd ::= COMMIT trans_opt. {sqlite3CommitTransaction(pParse);} cmd ::= END trans_opt. {sqlite3CommitTransaction(pParse);} cmd ::= ROLLBACK trans_opt. {sqlite3RollbackTransaction(pParse);} @@ -203,10 +218,11 @@ columnid(A) ::= nm(X). { // %fallback ID ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST COLUMNKW - CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXCLUSIVE EXPLAIN FAIL FOR + CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXPLAIN FAIL FOR IGNORE IMMEDIATE INITIALLY INSTEAD LIKE_KW MATCH NO PLAN QUERY KEY OF OFFSET PRAGMA RAISE RECURSIVE RELEASE REPLACE RESTRICT ROW - ROLLBACK SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITH WITHOUT + ROLLBACK SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL + WITH WITHOUT %ifdef SQLITE_OMIT_COMPOUND_SELECT EXCEPT INTERSECT UNION %endif SQLITE_OMIT_COMPOUND_SELECT @@ -873,7 +889,7 @@ expr(A) ::= VARIABLE(X). { ** that look like this: #1 #2 ... These terms refer to registers ** in the virtual machine. #N is the N-th register. */ if( pParse->nested==0 ){ - sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &X); + parserSyntaxError(pParse, &X); A.pExpr = 0; }else{ A.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0, &X); diff --git a/tool/mkkeywordhash.c b/tool/mkkeywordhash.c index 50585b1d09..044ebb6721 100644 --- a/tool/mkkeywordhash.c +++ b/tool/mkkeywordhash.c @@ -171,7 +171,6 @@ static Keyword aKeywordTable[] = { { "COLLATE", "TK_COLLATE", ALWAYS }, { "COLUMN", "TK_COLUMNKW", ALTER }, { "COMMIT", "TK_COMMIT", ALWAYS }, - { "CONCURRENT", "TK_CONCURRENT", ALWAYS }, { "CONFLICT", "TK_CONFLICT", CONFLICT }, { "CONSTRAINT", "TK_CONSTRAINT", ALWAYS }, { "CREATE", "TK_CREATE", ALWAYS }, @@ -193,7 +192,6 @@ static Keyword aKeywordTable[] = { { "ELSE", "TK_ELSE", ALWAYS }, { "ESCAPE", "TK_ESCAPE", ALWAYS }, { "EXCEPT", "TK_EXCEPT", COMPOUND }, - { "EXCLUSIVE", "TK_EXCLUSIVE", ALWAYS }, { "EXISTS", "TK_EXISTS", ALWAYS }, { "EXPLAIN", "TK_EXPLAIN", EXPLAIN }, { "FAIL", "TK_FAIL", CONFLICT|TRIGGER }, From b00083429adf5d73c78c803a2617f67d4e95371f Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 6 May 2016 21:04:47 +0000 Subject: [PATCH 031/179] Add test/bc_test1.c, for testing the degree of concurrency provided by this branch under various conditions. FossilOrigin-Name: 128c7eaed5479b615d75ebce1d781ea661e0fb2d --- main.mk | 3 + manifest | 18 +- manifest.uuid | 2 +- test/bc_test1.c | 137 +++++++ test/threadtest3.c | 867 +------------------------------------------- test/tt3_core.c | 883 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 1035 insertions(+), 875 deletions(-) create mode 100644 test/bc_test1.c create mode 100644 test/tt3_core.c diff --git a/main.mk b/main.mk index 1e791416d8..f4f6287753 100644 --- a/main.mk +++ b/main.mk @@ -819,6 +819,9 @@ THREADTEST3_SRC = $(TOP)/test/threadtest3.c \ threadtest3$(EXE): sqlite3.o $(THREADTEST3_SRC) $(TOP)/src/test_multiplex.c $(TCCX) $(TOP)/test/threadtest3.c $(TOP)/src/test_multiplex.c sqlite3.o -o $@ $(THREADLIB) +bc_test1$(EXE): sqlite3.o $(TOP)/test/bc_test1.c $(TOP)/test/tt3_core.c + $(TCCX) $(TOP)/test/bc_test1.c sqlite3.o -o $@ $(THREADLIB) + threadtest: threadtest3$(EXE) ./threadtest3$(EXE) diff --git a/manifest b/manifest index cc7dd90dc4..161feb3e40 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\senhancements\sfrom\strunk. -D 2016-04-29T16:01:18.190 +C Add\stest/bc_test1.c,\sfor\stesting\sthe\sdegree\sof\sconcurrency\sprovided\sby\sthis\sbranch\sunder\svarious\sconditions. +D 2016-05-06T21:04:47.598 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -302,7 +302,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk a283660f75c3c4b75d8c9d12a40fa38a066eee9d +F main.mk 968ad49af90277060b8ec7a15915273b76ee1d4d F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -521,6 +521,7 @@ F test/backup_malloc.test 7162d604ec2b4683c4b3799a48657fb8b5e2d450 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f F test/badutf2.test f5bc7f2d280670ecd79b9cf4f0f1760c607fe51f F test/bc_common.tcl b5e42d80305be95697e6370e015af571e5333a1c +F test/bc_test1.c d67cb1a933ae3a9116c8dc6da31e9ebf1c06ac23 F test/bestindex1.test 0cf1bd2d7b97d3a3a8c10736125274f64765c4ee F test/bestindex2.test 4a06b8922ab2fd09434870da8d1cdf525aaf7060 F test/between.test 34d375fb5ce1ae283ffe82b6b233e9f38e84fc6c @@ -1129,7 +1130,7 @@ F test/thread2.test f35d2106452b77523b3a2b7d1dcde2e5ee8f9e46 F test/thread_common.tcl 334639cadcb9f912bf82aa73f49efd5282e6cadd F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b F test/threadtest2.c a70a8e94bef23339d34226eb9521015ef99f4df8 -F test/threadtest3.c 38a612ea62854349ed66372f330a40d73c5cf956 +F test/threadtest3.c 4b413718ab3ef5e4f9f1ab0502b89707c5a9ccdc F test/threadtest4.c c1e67136ceb6c7ec8184e56ac61db28f96bd2925 F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 @@ -1294,6 +1295,7 @@ F test/triggerC.test 302d8995f5ffe63bbc15053abb3ef7a39cf5a092 F test/triggerD.test 8e7f3921a92a5797d472732108109e44575fa650 F test/triggerE.test 15fa63f1097db1f83dd62d121616006978063d1f F test/tt3_checkpoint.c 9e75cf7c1c364f52e1c47fd0f14c4340a9db0fe1 +F test/tt3_core.c 13c19feb4fe127bc86ae86d2c1237eb71c73f2ab F test/tt3_index.c 39eec10a35f57672225be4d182862152896dee4a F test/tt3_lookaside1.c 0377e202c3c2a50d688cb65ba203afeda6fafeb9 F test/tt3_stress.c c57d804716165811d979d4a719e05baccd79277f @@ -1489,7 +1491,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1f709fbf931e4d75848fc90532e2bc67ccd47cd4 9d0a5ae00273686ea35b43bc2ffaa8775c176363 -R 53ca713e0bf5b0eed86d88e36153628f -U drh -Z e98bb20abec63c17e483c921950d1a1a +P 91e5c07eaf884d3598df8eb54f0910a80df48397 +R 4bab1825e8924fd8c0c1d12f2130ddfc +U dan +Z 8e9093e5b020040f9bd55508cf282679 diff --git a/manifest.uuid b/manifest.uuid index 43d7e4a998..86543cee78 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -91e5c07eaf884d3598df8eb54f0910a80df48397 \ No newline at end of file +128c7eaed5479b615d75ebce1d781ea661e0fb2d \ No newline at end of file diff --git a/test/bc_test1.c b/test/bc_test1.c new file mode 100644 index 0000000000..dd8ab30440 --- /dev/null +++ b/test/bc_test1.c @@ -0,0 +1,137 @@ + + +#include + +#include + +#include "tt3_core.c" + + +typedef struct Config Config; + +struct Config { + int nIPT; /* --inserts-per-transaction */ + int nThread; /* --threads */ + int nSecond; /* --seconds */ + int bMutex; /* --mutex */ + + int bRm; /* --rm */ + sqlite3_mutex *pMutex; +}; + +static char *thread_main(int iTid, void *pArg){ + Config *pConfig = (Config*)pArg; + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + int nAttempt = 0; /* Attempted transactions */ + int nCommit = 0; /* Successful transactions */ + int j; + + opendb(&err, &db, "xyz.db", 0); + sqlite3_busy_handler(db.db, 0, 0); + execsql(&err, &db, "PRAGMA wal_autocheckpoint = 0"); + + while( !timetostop(&err) ){ + execsql(&err, &db, "BEGIN CONCURRENT"); + for(j=0; jnIPT; j++){ + execsql(&err, &db, + "INSERT INTO t1 VALUES" + "(randomblob(10), randomblob(20), randomblob(30), randomblob(200))" + ); + } + sqlite3_mutex_enter(pConfig->pMutex); + execsql(&err, &db, "COMMIT"); + sqlite3_mutex_leave(pConfig->pMutex); + nAttempt++; + if( err.rc==SQLITE_OK ){ + nCommit++; + }else{ + clear_error(&err, SQLITE_BUSY); + execsql(&err, &db, "ROLLBACK"); + } + } + + closedb(&err, &db); + return sqlite3_mprintf("%d/%d successful commits", nCommit, nAttempt); +} + +static void usage(char *zName){ + fprintf(stderr, "Usage: %s ?SWITCHES?\n", zName); + fprintf(stderr, "\n"); + fprintf(stderr, "where switches are\n"); + fprintf(stderr, " --seconds N\n"); + fprintf(stderr, " --inserts N\n"); + fprintf(stderr, " --threads N\n"); + fprintf(stderr, " --rm BOOL\n"); + fprintf(stderr, " --mutex BOOL\n"); + fprintf(stderr, "\n"); + exit(-1); +} + +int main(int argc, char **argv){ + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + Threadset threads = {0}; /* Test threads */ + Config sConfig = {5, 3, 5}; + int i; + + for(i=1; i=3 && 0==sqlite3_strnicmp(z, "--seconds", n) ){ + if( (++i)==argc ) usage(argv[0]); + sConfig.nSecond = atoi(argv[i]); + } + + else if( n>=3 && 0==sqlite3_strnicmp(z, "--inserts", n) ){ + if( (++i)==argc ) usage(argv[0]); + sConfig.nIPT = atoi(argv[i]); + } + + else if( n>=3 && 0==sqlite3_strnicmp(z, "--threads", n) ){ + if( (++i)==argc ) usage(argv[0]); + sConfig.nThread = atoi(argv[i]); + } + + else if( n>=3 && 0==sqlite3_strnicmp(z, "--rm", n) ){ + if( (++i)==argc ) usage(argv[0]); + sConfig.bRm = atoi(argv[i]); + } + + else if( n>=3 && 0==sqlite3_strnicmp(z, "--mutex", n) ){ + if( (++i)==argc ) usage(argv[0]); + sConfig.bMutex = atoi(argv[i]); + } + + else usage(argv[0]); + } + + printf("With: --threads %d --inserts %d --seconds %d --rm %d --mutex %d\n", + sConfig.nThread, sConfig.nIPT, sConfig.nSecond, sConfig.bRm, + sConfig.bMutex + ); + + /* Ensure the schema has been created */ + if( sConfig.bMutex ){ + sConfig.pMutex = sqlite3_mutex_alloc(SQLITE_MUTEX_RECURSIVE); + } + opendb(&err, &db, "xyz.db", sConfig.bRm); + + sql_script(&err, &db, + "PRAGMA journal_mode = wal;" + "CREATE TABLE IF NOT EXISTS t1(a PRIMARY KEY, b, c, d) WITHOUT ROWID;" + "CREATE INDEX IF NOT EXISTS t1b ON t1(b);" + "CREATE INDEX IF NOT EXISTS t1c ON t1(c);" + ); + closedb(&err, &db); + + setstoptime(&err, sConfig.nSecond*1000); + for(i=0; iiLine = ((e)->rc ? (e)->iLine : __LINE__)) - -/* Database functions */ -#define opendb(w,x,y,z) (SEL(w), opendb_x(w,x,y,z)) -#define closedb(y,z) (SEL(y), closedb_x(y,z)) - -/* Functions to execute SQL */ -#define sql_script(x,y,z) (SEL(x), sql_script_x(x,y,z)) -#define integrity_check(x,y) (SEL(x), integrity_check_x(x,y)) -#define execsql_i64(x,y,...) (SEL(x), execsql_i64_x(x,y,__VA_ARGS__)) -#define execsql_text(x,y,z,...) (SEL(x), execsql_text_x(x,y,z,__VA_ARGS__)) -#define execsql(x,y,...) (SEL(x), (void)execsql_i64_x(x,y,__VA_ARGS__)) -#define sql_script_printf(x,y,z,...) ( \ - SEL(x), sql_script_printf_x(x,y,z,__VA_ARGS__) \ -) - -/* Thread functions */ -#define launch_thread(w,x,y,z) (SEL(w), launch_thread_x(w,x,y,z)) -#define join_all_threads(y,z) (SEL(y), join_all_threads_x(y,z)) - -/* Timer functions */ -#define setstoptime(y,z) (SEL(y), setstoptime_x(y,z)) -#define timetostop(z) (SEL(z), timetostop_x(z)) - -/* Report/clear errors. */ -#define test_error(z, ...) test_error_x(z, sqlite3_mprintf(__VA_ARGS__)) -#define clear_error(y,z) clear_error_x(y, z) - -/* File-system operations */ -#define filesize(y,z) (SEL(y), filesize_x(y,z)) -#define filecopy(x,y,z) (SEL(x), filecopy_x(x,y,z)) - -#define PTR2INT(x) ((int)((intptr_t)x)) -#define INT2PTR(x) ((void*)((intptr_t)x)) - -/* -** End of test code/infrastructure interface macros. -*************************************************************************/ #include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include "test_multiplex.h" +#include "tt3_core.c" /* Required to link test_multiplex.c */ #ifndef SQLITE_OMIT_WSD int sqlite3PendingByte = 0x40000000; #endif -/* - * This code implements the MD5 message-digest algorithm. - * The algorithm is due to Ron Rivest. This code was - * written by Colin Plumb in 1993, no copyright is claimed. - * This code is in the public domain; do with it what you wish. - * - * Equivalent code is available from RSA Data Security, Inc. - * This code has been tested against that, and is equivalent, - * except that you don't need to include two pages of legalese - * with every copy. - * - * To compute the message digest of a chunk of bytes, declare an - * MD5Context structure, pass it to MD5Init, call MD5Update as - * needed on buffers full of bytes, and then call MD5Final, which - * will fill a supplied 16-byte array with the digest. - */ - -/* - * If compiled on a machine that doesn't have a 32-bit integer, - * you just set "uint32" to the appropriate datatype for an - * unsigned 32-bit integer. For example: - * - * cc -Duint32='unsigned long' md5.c - * - */ -#ifndef uint32 -# define uint32 unsigned int -#endif - -struct MD5Context { - int isInit; - uint32 buf[4]; - uint32 bits[2]; - union { - unsigned char in[64]; - uint32 in32[16]; - } u; -}; -typedef struct MD5Context MD5Context; - -/* - * Note: this code is harmless on little-endian machines. - */ -static void byteReverse (unsigned char *buf, unsigned longs){ - uint32 t; - do { - t = (uint32)((unsigned)buf[3]<<8 | buf[2]) << 16 | - ((unsigned)buf[1]<<8 | buf[0]); - *(uint32 *)buf = t; - buf += 4; - } while (--longs); -} -/* The four core functions - F1 is optimized somewhat */ - -/* #define F1(x, y, z) (x & y | ~x & z) */ -#define F1(x, y, z) (z ^ (x & (y ^ z))) -#define F2(x, y, z) F1(z, x, y) -#define F3(x, y, z) (x ^ y ^ z) -#define F4(x, y, z) (y ^ (x | ~z)) - -/* This is the central step in the MD5 algorithm. */ -#define MD5STEP(f, w, x, y, z, data, s) \ - ( w += f(x, y, z) + data, w = w<>(32-s), w += x ) - -/* - * The core of the MD5 algorithm, this alters an existing MD5 hash to - * reflect the addition of 16 longwords of new data. MD5Update blocks - * the data and converts bytes into longwords for this routine. - */ -static void MD5Transform(uint32 buf[4], const uint32 in[16]){ - register uint32 a, b, c, d; - - a = buf[0]; - b = buf[1]; - c = buf[2]; - d = buf[3]; - - MD5STEP(F1, a, b, c, d, in[ 0]+0xd76aa478, 7); - MD5STEP(F1, d, a, b, c, in[ 1]+0xe8c7b756, 12); - MD5STEP(F1, c, d, a, b, in[ 2]+0x242070db, 17); - MD5STEP(F1, b, c, d, a, in[ 3]+0xc1bdceee, 22); - MD5STEP(F1, a, b, c, d, in[ 4]+0xf57c0faf, 7); - MD5STEP(F1, d, a, b, c, in[ 5]+0x4787c62a, 12); - MD5STEP(F1, c, d, a, b, in[ 6]+0xa8304613, 17); - MD5STEP(F1, b, c, d, a, in[ 7]+0xfd469501, 22); - MD5STEP(F1, a, b, c, d, in[ 8]+0x698098d8, 7); - MD5STEP(F1, d, a, b, c, in[ 9]+0x8b44f7af, 12); - MD5STEP(F1, c, d, a, b, in[10]+0xffff5bb1, 17); - MD5STEP(F1, b, c, d, a, in[11]+0x895cd7be, 22); - MD5STEP(F1, a, b, c, d, in[12]+0x6b901122, 7); - MD5STEP(F1, d, a, b, c, in[13]+0xfd987193, 12); - MD5STEP(F1, c, d, a, b, in[14]+0xa679438e, 17); - MD5STEP(F1, b, c, d, a, in[15]+0x49b40821, 22); - - MD5STEP(F2, a, b, c, d, in[ 1]+0xf61e2562, 5); - MD5STEP(F2, d, a, b, c, in[ 6]+0xc040b340, 9); - MD5STEP(F2, c, d, a, b, in[11]+0x265e5a51, 14); - MD5STEP(F2, b, c, d, a, in[ 0]+0xe9b6c7aa, 20); - MD5STEP(F2, a, b, c, d, in[ 5]+0xd62f105d, 5); - MD5STEP(F2, d, a, b, c, in[10]+0x02441453, 9); - MD5STEP(F2, c, d, a, b, in[15]+0xd8a1e681, 14); - MD5STEP(F2, b, c, d, a, in[ 4]+0xe7d3fbc8, 20); - MD5STEP(F2, a, b, c, d, in[ 9]+0x21e1cde6, 5); - MD5STEP(F2, d, a, b, c, in[14]+0xc33707d6, 9); - MD5STEP(F2, c, d, a, b, in[ 3]+0xf4d50d87, 14); - MD5STEP(F2, b, c, d, a, in[ 8]+0x455a14ed, 20); - MD5STEP(F2, a, b, c, d, in[13]+0xa9e3e905, 5); - MD5STEP(F2, d, a, b, c, in[ 2]+0xfcefa3f8, 9); - MD5STEP(F2, c, d, a, b, in[ 7]+0x676f02d9, 14); - MD5STEP(F2, b, c, d, a, in[12]+0x8d2a4c8a, 20); - - MD5STEP(F3, a, b, c, d, in[ 5]+0xfffa3942, 4); - MD5STEP(F3, d, a, b, c, in[ 8]+0x8771f681, 11); - MD5STEP(F3, c, d, a, b, in[11]+0x6d9d6122, 16); - MD5STEP(F3, b, c, d, a, in[14]+0xfde5380c, 23); - MD5STEP(F3, a, b, c, d, in[ 1]+0xa4beea44, 4); - MD5STEP(F3, d, a, b, c, in[ 4]+0x4bdecfa9, 11); - MD5STEP(F3, c, d, a, b, in[ 7]+0xf6bb4b60, 16); - MD5STEP(F3, b, c, d, a, in[10]+0xbebfbc70, 23); - MD5STEP(F3, a, b, c, d, in[13]+0x289b7ec6, 4); - MD5STEP(F3, d, a, b, c, in[ 0]+0xeaa127fa, 11); - MD5STEP(F3, c, d, a, b, in[ 3]+0xd4ef3085, 16); - MD5STEP(F3, b, c, d, a, in[ 6]+0x04881d05, 23); - MD5STEP(F3, a, b, c, d, in[ 9]+0xd9d4d039, 4); - MD5STEP(F3, d, a, b, c, in[12]+0xe6db99e5, 11); - MD5STEP(F3, c, d, a, b, in[15]+0x1fa27cf8, 16); - MD5STEP(F3, b, c, d, a, in[ 2]+0xc4ac5665, 23); - - MD5STEP(F4, a, b, c, d, in[ 0]+0xf4292244, 6); - MD5STEP(F4, d, a, b, c, in[ 7]+0x432aff97, 10); - MD5STEP(F4, c, d, a, b, in[14]+0xab9423a7, 15); - MD5STEP(F4, b, c, d, a, in[ 5]+0xfc93a039, 21); - MD5STEP(F4, a, b, c, d, in[12]+0x655b59c3, 6); - MD5STEP(F4, d, a, b, c, in[ 3]+0x8f0ccc92, 10); - MD5STEP(F4, c, d, a, b, in[10]+0xffeff47d, 15); - MD5STEP(F4, b, c, d, a, in[ 1]+0x85845dd1, 21); - MD5STEP(F4, a, b, c, d, in[ 8]+0x6fa87e4f, 6); - MD5STEP(F4, d, a, b, c, in[15]+0xfe2ce6e0, 10); - MD5STEP(F4, c, d, a, b, in[ 6]+0xa3014314, 15); - MD5STEP(F4, b, c, d, a, in[13]+0x4e0811a1, 21); - MD5STEP(F4, a, b, c, d, in[ 4]+0xf7537e82, 6); - MD5STEP(F4, d, a, b, c, in[11]+0xbd3af235, 10); - MD5STEP(F4, c, d, a, b, in[ 2]+0x2ad7d2bb, 15); - MD5STEP(F4, b, c, d, a, in[ 9]+0xeb86d391, 21); - - buf[0] += a; - buf[1] += b; - buf[2] += c; - buf[3] += d; -} - -/* - * Start MD5 accumulation. Set bit count to 0 and buffer to mysterious - * initialization constants. - */ -static void MD5Init(MD5Context *ctx){ - ctx->isInit = 1; - ctx->buf[0] = 0x67452301; - ctx->buf[1] = 0xefcdab89; - ctx->buf[2] = 0x98badcfe; - ctx->buf[3] = 0x10325476; - ctx->bits[0] = 0; - ctx->bits[1] = 0; -} - -/* - * Update context to reflect the concatenation of another buffer full - * of bytes. - */ -static -void MD5Update(MD5Context *ctx, const unsigned char *buf, unsigned int len){ - uint32 t; - - /* Update bitcount */ - - t = ctx->bits[0]; - if ((ctx->bits[0] = t + ((uint32)len << 3)) < t) - ctx->bits[1]++; /* Carry from low to high */ - ctx->bits[1] += len >> 29; - - t = (t >> 3) & 0x3f; /* Bytes already in shsInfo->data */ - - /* Handle any leading odd-sized chunks */ - - if ( t ) { - unsigned char *p = (unsigned char *)ctx->u.in + t; - - t = 64-t; - if (len < t) { - memcpy(p, buf, len); - return; - } - memcpy(p, buf, t); - byteReverse(ctx->u.in, 16); - MD5Transform(ctx->buf, (uint32 *)ctx->u.in); - buf += t; - len -= t; - } - - /* Process data in 64-byte chunks */ - - while (len >= 64) { - memcpy(ctx->u.in, buf, 64); - byteReverse(ctx->u.in, 16); - MD5Transform(ctx->buf, (uint32 *)ctx->u.in); - buf += 64; - len -= 64; - } - - /* Handle any remaining bytes of data. */ - - memcpy(ctx->u.in, buf, len); -} - -/* - * Final wrapup - pad to 64-byte boundary with the bit pattern - * 1 0* (64-bit count of bits processed, MSB-first) - */ -static void MD5Final(unsigned char digest[16], MD5Context *ctx){ - unsigned count; - unsigned char *p; - - /* Compute number of bytes mod 64 */ - count = (ctx->bits[0] >> 3) & 0x3F; - - /* Set the first char of padding to 0x80. This is safe since there is - always at least one byte free */ - p = ctx->u.in + count; - *p++ = 0x80; - - /* Bytes of padding needed to make 64 bytes */ - count = 64 - 1 - count; - - /* Pad out to 56 mod 64 */ - if (count < 8) { - /* Two lots of padding: Pad the first block to 64 bytes */ - memset(p, 0, count); - byteReverse(ctx->u.in, 16); - MD5Transform(ctx->buf, (uint32 *)ctx->u.in); - - /* Now fill the next block with 56 bytes */ - memset(ctx->u.in, 0, 56); - } else { - /* Pad block to 56 bytes */ - memset(p, 0, count-8); - } - byteReverse(ctx->u.in, 14); - - /* Append length in bits and transform */ - ctx->u.in32[14] = ctx->bits[0]; - ctx->u.in32[15] = ctx->bits[1]; - - MD5Transform(ctx->buf, (uint32 *)ctx->u.in); - byteReverse((unsigned char *)ctx->buf, 4); - memcpy(digest, ctx->buf, 16); - memset(ctx, 0, sizeof(*ctx)); /* In case it is sensitive */ -} - -/* -** Convert a 128-bit MD5 digest into a 32-digit base-16 number. -*/ -static void MD5DigestToBase16(unsigned char *digest, char *zBuf){ - static char const zEncode[] = "0123456789abcdef"; - int i, j; - - for(j=i=0; i<16; i++){ - int a = digest[i]; - zBuf[j++] = zEncode[(a>>4)&0xf]; - zBuf[j++] = zEncode[a & 0xf]; - } - zBuf[j] = 0; -} - -/* -** During testing, the special md5sum() aggregate function is available. -** inside SQLite. The following routines implement that function. -*/ -static void md5step(sqlite3_context *context, int argc, sqlite3_value **argv){ - MD5Context *p; - int i; - if( argc<1 ) return; - p = sqlite3_aggregate_context(context, sizeof(*p)); - if( p==0 ) return; - if( !p->isInit ){ - MD5Init(p); - } - for(i=0; izErr); - p->zErr = 0; - p->rc = 0; -} - -static void print_err(Error *p){ - if( p->rc!=SQLITE_OK ){ - int isWarn = 0; - if( p->rc==SQLITE_SCHEMA ) isWarn = 1; - if( sqlite3_strglob("* - no such table: *",p->zErr)==0 ) isWarn = 1; - printf("%s: (%d) \"%s\" at line %d\n", isWarn ? "Warning" : "Error", - p->rc, p->zErr, p->iLine); - if( !isWarn ) nGlobalErr++; - fflush(stdout); - } -} - -static void print_and_free_err(Error *p){ - print_err(p); - free_err(p); -} - -static void system_error(Error *pErr, int iSys){ - pErr->rc = iSys; - pErr->zErr = (char *)sqlite3_malloc(512); - strerror_r(iSys, pErr->zErr, 512); - pErr->zErr[511] = '\0'; -} - -static void sqlite_error( - Error *pErr, - Sqlite *pDb, - const char *zFunc -){ - pErr->rc = sqlite3_errcode(pDb->db); - pErr->zErr = sqlite3_mprintf( - "sqlite3_%s() - %s (%d)", zFunc, sqlite3_errmsg(pDb->db), - sqlite3_extended_errcode(pDb->db) - ); -} - -static void test_error_x( - Error *pErr, - char *zErr -){ - if( pErr->rc==SQLITE_OK ){ - pErr->rc = 1; - pErr->zErr = zErr; - }else{ - sqlite3_free(zErr); - } -} - -static void clear_error_x( - Error *pErr, - int rc -){ - if( pErr->rc==rc ){ - pErr->rc = SQLITE_OK; - sqlite3_free(pErr->zErr); - pErr->zErr = 0; - } -} - -static int busyhandler(void *pArg, int n){ - usleep(10*1000); - return 1; -} - -static void opendb_x( - Error *pErr, /* IN/OUT: Error code */ - Sqlite *pDb, /* OUT: Database handle */ - const char *zFile, /* Database file name */ - int bDelete /* True to delete db file before opening */ -){ - if( pErr->rc==SQLITE_OK ){ - int rc; - int flags = SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE | SQLITE_OPEN_URI; - if( bDelete ) unlink(zFile); - rc = sqlite3_open_v2(zFile, &pDb->db, flags, 0); - if( rc ){ - sqlite_error(pErr, pDb, "open"); - sqlite3_close(pDb->db); - pDb->db = 0; - }else{ - sqlite3_create_function( - pDb->db, "md5sum", -1, SQLITE_UTF8, 0, 0, md5step, md5finalize - ); - sqlite3_busy_handler(pDb->db, busyhandler, 0); - sqlite3_exec(pDb->db, "PRAGMA synchronous=OFF", 0, 0, 0); - } - } -} - -static void closedb_x( - Error *pErr, /* IN/OUT: Error code */ - Sqlite *pDb /* OUT: Database handle */ -){ - int rc; - int i; - Statement *pIter; - Statement *pNext; - for(pIter=pDb->pCache; pIter; pIter=pNext){ - pNext = pIter->pNext; - sqlite3_finalize(pIter->pStmt); - sqlite3_free(pIter); - } - for(i=0; inText; i++){ - sqlite3_free(pDb->aText[i]); - } - sqlite3_free(pDb->aText); - rc = sqlite3_close(pDb->db); - if( rc && pErr->rc==SQLITE_OK ){ - pErr->zErr = sqlite3_mprintf("%s", sqlite3_errmsg(pDb->db)); - } - memset(pDb, 0, sizeof(Sqlite)); -} - -static void sql_script_x( - Error *pErr, /* IN/OUT: Error code */ - Sqlite *pDb, /* Database handle */ - const char *zSql /* SQL script to execute */ -){ - if( pErr->rc==SQLITE_OK ){ - pErr->rc = sqlite3_exec(pDb->db, zSql, 0, 0, &pErr->zErr); - } -} - -static void sql_script_printf_x( - Error *pErr, /* IN/OUT: Error code */ - Sqlite *pDb, /* Database handle */ - const char *zFormat, /* SQL printf format string */ - ... /* Printf args */ -){ - va_list ap; /* ... printf arguments */ - va_start(ap, zFormat); - if( pErr->rc==SQLITE_OK ){ - char *zSql = sqlite3_vmprintf(zFormat, ap); - pErr->rc = sqlite3_exec(pDb->db, zSql, 0, 0, &pErr->zErr); - sqlite3_free(zSql); - } - va_end(ap); -} - -static Statement *getSqlStatement( - Error *pErr, /* IN/OUT: Error code */ - Sqlite *pDb, /* Database handle */ - const char *zSql /* SQL statement */ -){ - Statement *pRet; - int rc; - - for(pRet=pDb->pCache; pRet; pRet=pRet->pNext){ - if( 0==strcmp(sqlite3_sql(pRet->pStmt), zSql) ){ - return pRet; - } - } - - pRet = sqlite3_malloc(sizeof(Statement)); - rc = sqlite3_prepare_v2(pDb->db, zSql, -1, &pRet->pStmt, 0); - if( rc!=SQLITE_OK ){ - sqlite_error(pErr, pDb, "prepare_v2"); - return 0; - } - assert( 0==strcmp(sqlite3_sql(pRet->pStmt), zSql) ); - - pRet->pNext = pDb->pCache; - pDb->pCache = pRet; - return pRet; -} - -static sqlite3_stmt *getAndBindSqlStatement( - Error *pErr, /* IN/OUT: Error code */ - Sqlite *pDb, /* Database handle */ - va_list ap /* SQL followed by parameters */ -){ - Statement *pStatement; /* The SQLite statement wrapper */ - sqlite3_stmt *pStmt; /* The SQLite statement to return */ - int i; /* Used to iterate through parameters */ - - pStatement = getSqlStatement(pErr, pDb, va_arg(ap, const char *)); - if( !pStatement ) return 0; - pStmt = pStatement->pStmt; - for(i=1; i<=sqlite3_bind_parameter_count(pStmt); i++){ - const char *zName = sqlite3_bind_parameter_name(pStmt, i); - void * pArg = va_arg(ap, void*); - - switch( zName[1] ){ - case 'i': - sqlite3_bind_int64(pStmt, i, *(i64 *)pArg); - break; - - default: - pErr->rc = 1; - pErr->zErr = sqlite3_mprintf("Cannot discern type: \"%s\"", zName); - pStmt = 0; - break; - } - } - - return pStmt; -} - -static i64 execsql_i64_x( - Error *pErr, /* IN/OUT: Error code */ - Sqlite *pDb, /* Database handle */ - ... /* SQL and pointers to parameter values */ -){ - i64 iRet = 0; - if( pErr->rc==SQLITE_OK ){ - sqlite3_stmt *pStmt; /* SQL statement to execute */ - va_list ap; /* ... arguments */ - va_start(ap, pDb); - pStmt = getAndBindSqlStatement(pErr, pDb, ap); - if( pStmt ){ - int first = 1; - while( SQLITE_ROW==sqlite3_step(pStmt) ){ - if( first && sqlite3_column_count(pStmt)>0 ){ - iRet = sqlite3_column_int64(pStmt, 0); - } - first = 0; - } - if( SQLITE_OK!=sqlite3_reset(pStmt) ){ - sqlite_error(pErr, pDb, "reset"); - } - } - va_end(ap); - } - return iRet; -} - -static char * execsql_text_x( - Error *pErr, /* IN/OUT: Error code */ - Sqlite *pDb, /* Database handle */ - int iSlot, /* Db handle slot to store text in */ - ... /* SQL and pointers to parameter values */ -){ - char *zRet = 0; - - if( iSlot>=pDb->nText ){ - int nByte = sizeof(char *)*(iSlot+1); - pDb->aText = (char **)sqlite3_realloc(pDb->aText, nByte); - memset(&pDb->aText[pDb->nText], 0, sizeof(char*)*(iSlot+1-pDb->nText)); - pDb->nText = iSlot+1; - } - - if( pErr->rc==SQLITE_OK ){ - sqlite3_stmt *pStmt; /* SQL statement to execute */ - va_list ap; /* ... arguments */ - va_start(ap, iSlot); - pStmt = getAndBindSqlStatement(pErr, pDb, ap); - if( pStmt ){ - int first = 1; - while( SQLITE_ROW==sqlite3_step(pStmt) ){ - if( first && sqlite3_column_count(pStmt)>0 ){ - zRet = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0)); - sqlite3_free(pDb->aText[iSlot]); - pDb->aText[iSlot] = zRet; - } - first = 0; - } - if( SQLITE_OK!=sqlite3_reset(pStmt) ){ - sqlite_error(pErr, pDb, "reset"); - } - } - va_end(ap); - } - - return zRet; -} - -static void integrity_check_x( - Error *pErr, /* IN/OUT: Error code */ - Sqlite *pDb /* Database handle */ -){ - if( pErr->rc==SQLITE_OK ){ - Statement *pStatement; /* Statement to execute */ - char *zErr = 0; /* Integrity check error */ - - pStatement = getSqlStatement(pErr, pDb, "PRAGMA integrity_check"); - if( pStatement ){ - sqlite3_stmt *pStmt = pStatement->pStmt; - while( SQLITE_ROW==sqlite3_step(pStmt) ){ - const char *z = (const char*)sqlite3_column_text(pStmt, 0); - if( strcmp(z, "ok") ){ - if( zErr==0 ){ - zErr = sqlite3_mprintf("%s", z); - }else{ - zErr = sqlite3_mprintf("%z\n%s", zErr, z); - } - } - } - sqlite3_reset(pStmt); - - if( zErr ){ - pErr->zErr = zErr; - pErr->rc = 1; - } - } - } -} - -static void *launch_thread_main(void *pArg){ - Thread *p = (Thread *)pArg; - return (void *)p->xProc(p->iTid, p->pArg); -} - -static void launch_thread_x( - Error *pErr, /* IN/OUT: Error code */ - Threadset *pThreads, /* Thread set */ - char *(*xProc)(int, void*), /* Proc to run */ - void *pArg /* Argument passed to thread proc */ -){ - if( pErr->rc==SQLITE_OK ){ - int iTid = ++pThreads->iMaxTid; - Thread *p; - int rc; - - p = (Thread *)sqlite3_malloc(sizeof(Thread)); - memset(p, 0, sizeof(Thread)); - p->iTid = iTid; - p->pArg = pArg; - p->xProc = xProc; - - rc = pthread_create(&p->tid, NULL, launch_thread_main, (void *)p); - if( rc!=0 ){ - system_error(pErr, rc); - sqlite3_free(p); - }else{ - p->pNext = pThreads->pThread; - pThreads->pThread = p; - } - } -} - -static void join_all_threads_x( - Error *pErr, /* IN/OUT: Error code */ - Threadset *pThreads /* Thread set */ -){ - Thread *p; - Thread *pNext; - for(p=pThreads->pThread; p; p=pNext){ - void *ret; - pNext = p->pNext; - int rc; - rc = pthread_join(p->tid, &ret); - if( rc!=0 ){ - if( pErr->rc==SQLITE_OK ) system_error(pErr, rc); - }else{ - printf("Thread %d says: %s\n", p->iTid, (ret==0 ? "..." : (char *)ret)); - fflush(stdout); - } - sqlite3_free(p); - } - pThreads->pThread = 0; -} - -static i64 filesize_x( - Error *pErr, - const char *zFile -){ - i64 iRet = 0; - if( pErr->rc==SQLITE_OK ){ - struct stat sStat; - if( stat(zFile, &sStat) ){ - iRet = -1; - }else{ - iRet = sStat.st_size; - } - } - return iRet; -} - -static void filecopy_x( - Error *pErr, - const char *zFrom, - const char *zTo -){ - if( pErr->rc==SQLITE_OK ){ - i64 nByte = filesize_x(pErr, zFrom); - if( nByte<0 ){ - test_error_x(pErr, sqlite3_mprintf("no such file: %s", zFrom)); - }else{ - i64 iOff; - char aBuf[1024]; - int fd1; - int fd2; - unlink(zTo); - - fd1 = open(zFrom, O_RDONLY); - if( fd1<0 ){ - system_error(pErr, errno); - return; - } - fd2 = open(zTo, O_RDWR|O_CREAT|O_EXCL, 0644); - if( fd2<0 ){ - system_error(pErr, errno); - close(fd1); - return; - } - - iOff = 0; - while( iOffnByte ){ - nCopy = nByte - iOff; - } - if( nCopy!=read(fd1, aBuf, nCopy) ){ - system_error(pErr, errno); - break; - } - if( nCopy!=write(fd2, aBuf, nCopy) ){ - system_error(pErr, errno); - break; - } - iOff += nCopy; - } - - close(fd1); - close(fd2); - } - } -} - -/* -** Used by setstoptime() and timetostop(). -*/ -static double timelimit = 0.0; - -static double currentTime(void){ - double t; - static sqlite3_vfs *pTimelimitVfs = 0; - if( pTimelimitVfs==0 ) pTimelimitVfs = sqlite3_vfs_find(0); - if( pTimelimitVfs->iVersion>=2 && pTimelimitVfs->xCurrentTimeInt64!=0 ){ - sqlite3_int64 tm; - pTimelimitVfs->xCurrentTimeInt64(pTimelimitVfs, &tm); - t = tm/86400000.0; - }else{ - pTimelimitVfs->xCurrentTime(pTimelimitVfs, &t); - } - return t; -} - -static void setstoptime_x( - Error *pErr, /* IN/OUT: Error code */ - int nMs /* Milliseconds until "stop time" */ -){ - if( pErr->rc==SQLITE_OK ){ - double t = currentTime(); - timelimit = t + ((double)nMs)/(1000.0*60.0*60.0*24.0); - } -} - -static int timetostop_x( - Error *pErr /* IN/OUT: Error code */ -){ - int ret = 1; - if( pErr->rc==SQLITE_OK ){ - double t = currentTime(); - ret = (t >= timelimit); - } - return ret; -} - /************************************************************************* ************************************************************************** diff --git a/test/tt3_core.c b/test/tt3_core.c new file mode 100644 index 0000000000..21de432e98 --- /dev/null +++ b/test/tt3_core.c @@ -0,0 +1,883 @@ +/* +** 2016-05-07 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* +** The "Set Error Line" macro. +*/ +#define SEL(e) ((e)->iLine = ((e)->rc ? (e)->iLine : __LINE__)) + +/* Database functions */ +#define opendb(w,x,y,z) (SEL(w), opendb_x(w,x,y,z)) +#define closedb(y,z) (SEL(y), closedb_x(y,z)) + +/* Functions to execute SQL */ +#define sql_script(x,y,z) (SEL(x), sql_script_x(x,y,z)) +#define integrity_check(x,y) (SEL(x), integrity_check_x(x,y)) +#define execsql_i64(x,y,...) (SEL(x), execsql_i64_x(x,y,__VA_ARGS__)) +#define execsql_text(x,y,z,...) (SEL(x), execsql_text_x(x,y,z,__VA_ARGS__)) +#define execsql(x,y,...) (SEL(x), (void)execsql_i64_x(x,y,__VA_ARGS__)) +#define sql_script_printf(x,y,z,...) ( \ + SEL(x), sql_script_printf_x(x,y,z,__VA_ARGS__) \ +) + +/* Thread functions */ +#define launch_thread(w,x,y,z) (SEL(w), launch_thread_x(w,x,y,z)) +#define join_all_threads(y,z) (SEL(y), join_all_threads_x(y,z)) + +/* Timer functions */ +#define setstoptime(y,z) (SEL(y), setstoptime_x(y,z)) +#define timetostop(z) (SEL(z), timetostop_x(z)) + +/* Report/clear errors. */ +#define test_error(z, ...) test_error_x(z, sqlite3_mprintf(__VA_ARGS__)) +#define clear_error(y,z) clear_error_x(y, z) + +/* File-system operations */ +#define filesize(y,z) (SEL(y), filesize_x(y,z)) +#define filecopy(x,y,z) (SEL(x), filecopy_x(x,y,z)) + +#define PTR2INT(x) ((int)((intptr_t)x)) +#define INT2PTR(x) ((void*)((intptr_t)x)) + +/* +** End of test code/infrastructure interface macros. +*************************************************************************/ + +/* + * This code implements the MD5 message-digest algorithm. + * The algorithm is due to Ron Rivest. This code was + * written by Colin Plumb in 1993, no copyright is claimed. + * This code is in the public domain; do with it what you wish. + * + * Equivalent code is available from RSA Data Security, Inc. + * This code has been tested against that, and is equivalent, + * except that you don't need to include two pages of legalese + * with every copy. + * + * To compute the message digest of a chunk of bytes, declare an + * MD5Context structure, pass it to MD5Init, call MD5Update as + * needed on buffers full of bytes, and then call MD5Final, which + * will fill a supplied 16-byte array with the digest. + */ + +/* + * If compiled on a machine that doesn't have a 32-bit integer, + * you just set "uint32" to the appropriate datatype for an + * unsigned 32-bit integer. For example: + * + * cc -Duint32='unsigned long' md5.c + * + */ +#ifndef uint32 +# define uint32 unsigned int +#endif + +struct MD5Context { + int isInit; + uint32 buf[4]; + uint32 bits[2]; + union { + unsigned char in[64]; + uint32 in32[16]; + } u; +}; +typedef struct MD5Context MD5Context; + +/* + * Note: this code is harmless on little-endian machines. + */ +static void byteReverse (unsigned char *buf, unsigned longs){ + uint32 t; + do { + t = (uint32)((unsigned)buf[3]<<8 | buf[2]) << 16 | + ((unsigned)buf[1]<<8 | buf[0]); + *(uint32 *)buf = t; + buf += 4; + } while (--longs); +} +/* The four core functions - F1 is optimized somewhat */ + +/* #define F1(x, y, z) (x & y | ~x & z) */ +#define F1(x, y, z) (z ^ (x & (y ^ z))) +#define F2(x, y, z) F1(z, x, y) +#define F3(x, y, z) (x ^ y ^ z) +#define F4(x, y, z) (y ^ (x | ~z)) + +/* This is the central step in the MD5 algorithm. */ +#define MD5STEP(f, w, x, y, z, data, s) \ + ( w += f(x, y, z) + data, w = w<>(32-s), w += x ) + +/* + * The core of the MD5 algorithm, this alters an existing MD5 hash to + * reflect the addition of 16 longwords of new data. MD5Update blocks + * the data and converts bytes into longwords for this routine. + */ +static void MD5Transform(uint32 buf[4], const uint32 in[16]){ + register uint32 a, b, c, d; + + a = buf[0]; + b = buf[1]; + c = buf[2]; + d = buf[3]; + + MD5STEP(F1, a, b, c, d, in[ 0]+0xd76aa478, 7); + MD5STEP(F1, d, a, b, c, in[ 1]+0xe8c7b756, 12); + MD5STEP(F1, c, d, a, b, in[ 2]+0x242070db, 17); + MD5STEP(F1, b, c, d, a, in[ 3]+0xc1bdceee, 22); + MD5STEP(F1, a, b, c, d, in[ 4]+0xf57c0faf, 7); + MD5STEP(F1, d, a, b, c, in[ 5]+0x4787c62a, 12); + MD5STEP(F1, c, d, a, b, in[ 6]+0xa8304613, 17); + MD5STEP(F1, b, c, d, a, in[ 7]+0xfd469501, 22); + MD5STEP(F1, a, b, c, d, in[ 8]+0x698098d8, 7); + MD5STEP(F1, d, a, b, c, in[ 9]+0x8b44f7af, 12); + MD5STEP(F1, c, d, a, b, in[10]+0xffff5bb1, 17); + MD5STEP(F1, b, c, d, a, in[11]+0x895cd7be, 22); + MD5STEP(F1, a, b, c, d, in[12]+0x6b901122, 7); + MD5STEP(F1, d, a, b, c, in[13]+0xfd987193, 12); + MD5STEP(F1, c, d, a, b, in[14]+0xa679438e, 17); + MD5STEP(F1, b, c, d, a, in[15]+0x49b40821, 22); + + MD5STEP(F2, a, b, c, d, in[ 1]+0xf61e2562, 5); + MD5STEP(F2, d, a, b, c, in[ 6]+0xc040b340, 9); + MD5STEP(F2, c, d, a, b, in[11]+0x265e5a51, 14); + MD5STEP(F2, b, c, d, a, in[ 0]+0xe9b6c7aa, 20); + MD5STEP(F2, a, b, c, d, in[ 5]+0xd62f105d, 5); + MD5STEP(F2, d, a, b, c, in[10]+0x02441453, 9); + MD5STEP(F2, c, d, a, b, in[15]+0xd8a1e681, 14); + MD5STEP(F2, b, c, d, a, in[ 4]+0xe7d3fbc8, 20); + MD5STEP(F2, a, b, c, d, in[ 9]+0x21e1cde6, 5); + MD5STEP(F2, d, a, b, c, in[14]+0xc33707d6, 9); + MD5STEP(F2, c, d, a, b, in[ 3]+0xf4d50d87, 14); + MD5STEP(F2, b, c, d, a, in[ 8]+0x455a14ed, 20); + MD5STEP(F2, a, b, c, d, in[13]+0xa9e3e905, 5); + MD5STEP(F2, d, a, b, c, in[ 2]+0xfcefa3f8, 9); + MD5STEP(F2, c, d, a, b, in[ 7]+0x676f02d9, 14); + MD5STEP(F2, b, c, d, a, in[12]+0x8d2a4c8a, 20); + + MD5STEP(F3, a, b, c, d, in[ 5]+0xfffa3942, 4); + MD5STEP(F3, d, a, b, c, in[ 8]+0x8771f681, 11); + MD5STEP(F3, c, d, a, b, in[11]+0x6d9d6122, 16); + MD5STEP(F3, b, c, d, a, in[14]+0xfde5380c, 23); + MD5STEP(F3, a, b, c, d, in[ 1]+0xa4beea44, 4); + MD5STEP(F3, d, a, b, c, in[ 4]+0x4bdecfa9, 11); + MD5STEP(F3, c, d, a, b, in[ 7]+0xf6bb4b60, 16); + MD5STEP(F3, b, c, d, a, in[10]+0xbebfbc70, 23); + MD5STEP(F3, a, b, c, d, in[13]+0x289b7ec6, 4); + MD5STEP(F3, d, a, b, c, in[ 0]+0xeaa127fa, 11); + MD5STEP(F3, c, d, a, b, in[ 3]+0xd4ef3085, 16); + MD5STEP(F3, b, c, d, a, in[ 6]+0x04881d05, 23); + MD5STEP(F3, a, b, c, d, in[ 9]+0xd9d4d039, 4); + MD5STEP(F3, d, a, b, c, in[12]+0xe6db99e5, 11); + MD5STEP(F3, c, d, a, b, in[15]+0x1fa27cf8, 16); + MD5STEP(F3, b, c, d, a, in[ 2]+0xc4ac5665, 23); + + MD5STEP(F4, a, b, c, d, in[ 0]+0xf4292244, 6); + MD5STEP(F4, d, a, b, c, in[ 7]+0x432aff97, 10); + MD5STEP(F4, c, d, a, b, in[14]+0xab9423a7, 15); + MD5STEP(F4, b, c, d, a, in[ 5]+0xfc93a039, 21); + MD5STEP(F4, a, b, c, d, in[12]+0x655b59c3, 6); + MD5STEP(F4, d, a, b, c, in[ 3]+0x8f0ccc92, 10); + MD5STEP(F4, c, d, a, b, in[10]+0xffeff47d, 15); + MD5STEP(F4, b, c, d, a, in[ 1]+0x85845dd1, 21); + MD5STEP(F4, a, b, c, d, in[ 8]+0x6fa87e4f, 6); + MD5STEP(F4, d, a, b, c, in[15]+0xfe2ce6e0, 10); + MD5STEP(F4, c, d, a, b, in[ 6]+0xa3014314, 15); + MD5STEP(F4, b, c, d, a, in[13]+0x4e0811a1, 21); + MD5STEP(F4, a, b, c, d, in[ 4]+0xf7537e82, 6); + MD5STEP(F4, d, a, b, c, in[11]+0xbd3af235, 10); + MD5STEP(F4, c, d, a, b, in[ 2]+0x2ad7d2bb, 15); + MD5STEP(F4, b, c, d, a, in[ 9]+0xeb86d391, 21); + + buf[0] += a; + buf[1] += b; + buf[2] += c; + buf[3] += d; +} + +/* + * Start MD5 accumulation. Set bit count to 0 and buffer to mysterious + * initialization constants. + */ +static void MD5Init(MD5Context *ctx){ + ctx->isInit = 1; + ctx->buf[0] = 0x67452301; + ctx->buf[1] = 0xefcdab89; + ctx->buf[2] = 0x98badcfe; + ctx->buf[3] = 0x10325476; + ctx->bits[0] = 0; + ctx->bits[1] = 0; +} + +/* + * Update context to reflect the concatenation of another buffer full + * of bytes. + */ +static +void MD5Update(MD5Context *ctx, const unsigned char *buf, unsigned int len){ + uint32 t; + + /* Update bitcount */ + + t = ctx->bits[0]; + if ((ctx->bits[0] = t + ((uint32)len << 3)) < t) + ctx->bits[1]++; /* Carry from low to high */ + ctx->bits[1] += len >> 29; + + t = (t >> 3) & 0x3f; /* Bytes already in shsInfo->data */ + + /* Handle any leading odd-sized chunks */ + + if ( t ) { + unsigned char *p = (unsigned char *)ctx->u.in + t; + + t = 64-t; + if (len < t) { + memcpy(p, buf, len); + return; + } + memcpy(p, buf, t); + byteReverse(ctx->u.in, 16); + MD5Transform(ctx->buf, (uint32 *)ctx->u.in); + buf += t; + len -= t; + } + + /* Process data in 64-byte chunks */ + + while (len >= 64) { + memcpy(ctx->u.in, buf, 64); + byteReverse(ctx->u.in, 16); + MD5Transform(ctx->buf, (uint32 *)ctx->u.in); + buf += 64; + len -= 64; + } + + /* Handle any remaining bytes of data. */ + + memcpy(ctx->u.in, buf, len); +} + +/* + * Final wrapup - pad to 64-byte boundary with the bit pattern + * 1 0* (64-bit count of bits processed, MSB-first) + */ +static void MD5Final(unsigned char digest[16], MD5Context *ctx){ + unsigned count; + unsigned char *p; + + /* Compute number of bytes mod 64 */ + count = (ctx->bits[0] >> 3) & 0x3F; + + /* Set the first char of padding to 0x80. This is safe since there is + always at least one byte free */ + p = ctx->u.in + count; + *p++ = 0x80; + + /* Bytes of padding needed to make 64 bytes */ + count = 64 - 1 - count; + + /* Pad out to 56 mod 64 */ + if (count < 8) { + /* Two lots of padding: Pad the first block to 64 bytes */ + memset(p, 0, count); + byteReverse(ctx->u.in, 16); + MD5Transform(ctx->buf, (uint32 *)ctx->u.in); + + /* Now fill the next block with 56 bytes */ + memset(ctx->u.in, 0, 56); + } else { + /* Pad block to 56 bytes */ + memset(p, 0, count-8); + } + byteReverse(ctx->u.in, 14); + + /* Append length in bits and transform */ + ctx->u.in32[14] = ctx->bits[0]; + ctx->u.in32[15] = ctx->bits[1]; + + MD5Transform(ctx->buf, (uint32 *)ctx->u.in); + byteReverse((unsigned char *)ctx->buf, 4); + memcpy(digest, ctx->buf, 16); + memset(ctx, 0, sizeof(*ctx)); /* In case it is sensitive */ +} + +/* +** Convert a 128-bit MD5 digest into a 32-digit base-16 number. +*/ +static void MD5DigestToBase16(unsigned char *digest, char *zBuf){ + static char const zEncode[] = "0123456789abcdef"; + int i, j; + + for(j=i=0; i<16; i++){ + int a = digest[i]; + zBuf[j++] = zEncode[(a>>4)&0xf]; + zBuf[j++] = zEncode[a & 0xf]; + } + zBuf[j] = 0; +} + +/* +** During testing, the special md5sum() aggregate function is available. +** inside SQLite. The following routines implement that function. +*/ +static void md5step(sqlite3_context *context, int argc, sqlite3_value **argv){ + MD5Context *p; + int i; + if( argc<1 ) return; + p = sqlite3_aggregate_context(context, sizeof(*p)); + if( p==0 ) return; + if( !p->isInit ){ + MD5Init(p); + } + for(i=0; izErr); + p->zErr = 0; + p->rc = 0; +} + +static void print_err(Error *p){ + if( p->rc!=SQLITE_OK ){ + int isWarn = 0; + if( p->rc==SQLITE_SCHEMA ) isWarn = 1; + if( sqlite3_strglob("* - no such table: *",p->zErr)==0 ) isWarn = 1; + printf("%s: (%d) \"%s\" at line %d\n", isWarn ? "Warning" : "Error", + p->rc, p->zErr, p->iLine); + if( !isWarn ) nGlobalErr++; + fflush(stdout); + } +} + +static void print_and_free_err(Error *p){ + print_err(p); + free_err(p); +} + +static void system_error(Error *pErr, int iSys){ + pErr->rc = iSys; + pErr->zErr = (char *)sqlite3_malloc(512); + strerror_r(iSys, pErr->zErr, 512); + pErr->zErr[511] = '\0'; +} + +static void sqlite_error( + Error *pErr, + Sqlite *pDb, + const char *zFunc +){ + pErr->rc = sqlite3_errcode(pDb->db); + pErr->zErr = sqlite3_mprintf( + "sqlite3_%s() - %s (%d)", zFunc, sqlite3_errmsg(pDb->db), + sqlite3_extended_errcode(pDb->db) + ); +} + +static void test_error_x( + Error *pErr, + char *zErr +){ + if( pErr->rc==SQLITE_OK ){ + pErr->rc = 1; + pErr->zErr = zErr; + }else{ + sqlite3_free(zErr); + } +} + +static void clear_error_x( + Error *pErr, + int rc +){ + if( pErr->rc==rc ){ + pErr->rc = SQLITE_OK; + sqlite3_free(pErr->zErr); + pErr->zErr = 0; + } +} + +static int busyhandler(void *pArg, int n){ + usleep(10*1000); + return 1; +} + +static void opendb_x( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb, /* OUT: Database handle */ + const char *zFile, /* Database file name */ + int bDelete /* True to delete db file before opening */ +){ + if( pErr->rc==SQLITE_OK ){ + int rc; + int flags = SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE | SQLITE_OPEN_URI; + if( bDelete ) unlink(zFile); + rc = sqlite3_open_v2(zFile, &pDb->db, flags, 0); + if( rc ){ + sqlite_error(pErr, pDb, "open"); + sqlite3_close(pDb->db); + pDb->db = 0; + }else{ + sqlite3_create_function( + pDb->db, "md5sum", -1, SQLITE_UTF8, 0, 0, md5step, md5finalize + ); + sqlite3_busy_handler(pDb->db, busyhandler, 0); + sqlite3_exec(pDb->db, "PRAGMA synchronous=OFF", 0, 0, 0); + } + } +} + +static void closedb_x( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb /* OUT: Database handle */ +){ + int rc; + int i; + Statement *pIter; + Statement *pNext; + for(pIter=pDb->pCache; pIter; pIter=pNext){ + pNext = pIter->pNext; + sqlite3_finalize(pIter->pStmt); + sqlite3_free(pIter); + } + for(i=0; inText; i++){ + sqlite3_free(pDb->aText[i]); + } + sqlite3_free(pDb->aText); + rc = sqlite3_close(pDb->db); + if( rc && pErr->rc==SQLITE_OK ){ + pErr->zErr = sqlite3_mprintf("%s", sqlite3_errmsg(pDb->db)); + } + memset(pDb, 0, sizeof(Sqlite)); +} + +static void sql_script_x( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb, /* Database handle */ + const char *zSql /* SQL script to execute */ +){ + if( pErr->rc==SQLITE_OK ){ + pErr->rc = sqlite3_exec(pDb->db, zSql, 0, 0, &pErr->zErr); + } +} + +static void sql_script_printf_x( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb, /* Database handle */ + const char *zFormat, /* SQL printf format string */ + ... /* Printf args */ +){ + va_list ap; /* ... printf arguments */ + va_start(ap, zFormat); + if( pErr->rc==SQLITE_OK ){ + char *zSql = sqlite3_vmprintf(zFormat, ap); + pErr->rc = sqlite3_exec(pDb->db, zSql, 0, 0, &pErr->zErr); + sqlite3_free(zSql); + } + va_end(ap); +} + +static Statement *getSqlStatement( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb, /* Database handle */ + const char *zSql /* SQL statement */ +){ + Statement *pRet; + int rc; + + for(pRet=pDb->pCache; pRet; pRet=pRet->pNext){ + if( 0==strcmp(sqlite3_sql(pRet->pStmt), zSql) ){ + return pRet; + } + } + + pRet = sqlite3_malloc(sizeof(Statement)); + rc = sqlite3_prepare_v2(pDb->db, zSql, -1, &pRet->pStmt, 0); + if( rc!=SQLITE_OK ){ + sqlite_error(pErr, pDb, "prepare_v2"); + return 0; + } + assert( 0==strcmp(sqlite3_sql(pRet->pStmt), zSql) ); + + pRet->pNext = pDb->pCache; + pDb->pCache = pRet; + return pRet; +} + +static sqlite3_stmt *getAndBindSqlStatement( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb, /* Database handle */ + va_list ap /* SQL followed by parameters */ +){ + Statement *pStatement; /* The SQLite statement wrapper */ + sqlite3_stmt *pStmt; /* The SQLite statement to return */ + int i; /* Used to iterate through parameters */ + + pStatement = getSqlStatement(pErr, pDb, va_arg(ap, const char *)); + if( !pStatement ) return 0; + pStmt = pStatement->pStmt; + for(i=1; i<=sqlite3_bind_parameter_count(pStmt); i++){ + const char *zName = sqlite3_bind_parameter_name(pStmt, i); + void * pArg = va_arg(ap, void*); + + switch( zName[1] ){ + case 'i': + sqlite3_bind_int64(pStmt, i, *(i64 *)pArg); + break; + + default: + pErr->rc = 1; + pErr->zErr = sqlite3_mprintf("Cannot discern type: \"%s\"", zName); + pStmt = 0; + break; + } + } + + return pStmt; +} + +static i64 execsql_i64_x( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb, /* Database handle */ + ... /* SQL and pointers to parameter values */ +){ + i64 iRet = 0; + if( pErr->rc==SQLITE_OK ){ + sqlite3_stmt *pStmt; /* SQL statement to execute */ + va_list ap; /* ... arguments */ + va_start(ap, pDb); + pStmt = getAndBindSqlStatement(pErr, pDb, ap); + if( pStmt ){ + int first = 1; + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( first && sqlite3_column_count(pStmt)>0 ){ + iRet = sqlite3_column_int64(pStmt, 0); + } + first = 0; + } + if( SQLITE_OK!=sqlite3_reset(pStmt) ){ + sqlite_error(pErr, pDb, "reset"); + } + } + va_end(ap); + } + return iRet; +} + +static char * execsql_text_x( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb, /* Database handle */ + int iSlot, /* Db handle slot to store text in */ + ... /* SQL and pointers to parameter values */ +){ + char *zRet = 0; + + if( iSlot>=pDb->nText ){ + int nByte = sizeof(char *)*(iSlot+1); + pDb->aText = (char **)sqlite3_realloc(pDb->aText, nByte); + memset(&pDb->aText[pDb->nText], 0, sizeof(char*)*(iSlot+1-pDb->nText)); + pDb->nText = iSlot+1; + } + + if( pErr->rc==SQLITE_OK ){ + sqlite3_stmt *pStmt; /* SQL statement to execute */ + va_list ap; /* ... arguments */ + va_start(ap, iSlot); + pStmt = getAndBindSqlStatement(pErr, pDb, ap); + if( pStmt ){ + int first = 1; + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( first && sqlite3_column_count(pStmt)>0 ){ + zRet = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0)); + sqlite3_free(pDb->aText[iSlot]); + pDb->aText[iSlot] = zRet; + } + first = 0; + } + if( SQLITE_OK!=sqlite3_reset(pStmt) ){ + sqlite_error(pErr, pDb, "reset"); + } + } + va_end(ap); + } + + return zRet; +} + +static void integrity_check_x( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb /* Database handle */ +){ + if( pErr->rc==SQLITE_OK ){ + Statement *pStatement; /* Statement to execute */ + char *zErr = 0; /* Integrity check error */ + + pStatement = getSqlStatement(pErr, pDb, "PRAGMA integrity_check"); + if( pStatement ){ + sqlite3_stmt *pStmt = pStatement->pStmt; + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *z = (const char*)sqlite3_column_text(pStmt, 0); + if( strcmp(z, "ok") ){ + if( zErr==0 ){ + zErr = sqlite3_mprintf("%s", z); + }else{ + zErr = sqlite3_mprintf("%z\n%s", zErr, z); + } + } + } + sqlite3_reset(pStmt); + + if( zErr ){ + pErr->zErr = zErr; + pErr->rc = 1; + } + } + } +} + +static void *launch_thread_main(void *pArg){ + Thread *p = (Thread *)pArg; + return (void *)p->xProc(p->iTid, p->pArg); +} + +static void launch_thread_x( + Error *pErr, /* IN/OUT: Error code */ + Threadset *pThreads, /* Thread set */ + char *(*xProc)(int, void*), /* Proc to run */ + void *pArg /* Argument passed to thread proc */ +){ + if( pErr->rc==SQLITE_OK ){ + int iTid = ++pThreads->iMaxTid; + Thread *p; + int rc; + + p = (Thread *)sqlite3_malloc(sizeof(Thread)); + memset(p, 0, sizeof(Thread)); + p->iTid = iTid; + p->pArg = pArg; + p->xProc = xProc; + + rc = pthread_create(&p->tid, NULL, launch_thread_main, (void *)p); + if( rc!=0 ){ + system_error(pErr, rc); + sqlite3_free(p); + }else{ + p->pNext = pThreads->pThread; + pThreads->pThread = p; + } + } +} + +static void join_all_threads_x( + Error *pErr, /* IN/OUT: Error code */ + Threadset *pThreads /* Thread set */ +){ + Thread *p; + Thread *pNext; + for(p=pThreads->pThread; p; p=pNext){ + void *ret; + pNext = p->pNext; + int rc; + rc = pthread_join(p->tid, &ret); + if( rc!=0 ){ + if( pErr->rc==SQLITE_OK ) system_error(pErr, rc); + }else{ + printf("Thread %d says: %s\n", p->iTid, (ret==0 ? "..." : (char *)ret)); + fflush(stdout); + } + sqlite3_free(p); + } + pThreads->pThread = 0; +} + +static i64 filesize_x( + Error *pErr, + const char *zFile +){ + i64 iRet = 0; + if( pErr->rc==SQLITE_OK ){ + struct stat sStat; + if( stat(zFile, &sStat) ){ + iRet = -1; + }else{ + iRet = sStat.st_size; + } + } + return iRet; +} + +static void filecopy_x( + Error *pErr, + const char *zFrom, + const char *zTo +){ + if( pErr->rc==SQLITE_OK ){ + i64 nByte = filesize_x(pErr, zFrom); + if( nByte<0 ){ + test_error_x(pErr, sqlite3_mprintf("no such file: %s", zFrom)); + }else{ + i64 iOff; + char aBuf[1024]; + int fd1; + int fd2; + unlink(zTo); + + fd1 = open(zFrom, O_RDONLY); + if( fd1<0 ){ + system_error(pErr, errno); + return; + } + fd2 = open(zTo, O_RDWR|O_CREAT|O_EXCL, 0644); + if( fd2<0 ){ + system_error(pErr, errno); + close(fd1); + return; + } + + iOff = 0; + while( iOffnByte ){ + nCopy = nByte - iOff; + } + if( nCopy!=read(fd1, aBuf, nCopy) ){ + system_error(pErr, errno); + break; + } + if( nCopy!=write(fd2, aBuf, nCopy) ){ + system_error(pErr, errno); + break; + } + iOff += nCopy; + } + + close(fd1); + close(fd2); + } + } +} + +/* +** Used by setstoptime() and timetostop(). +*/ +static double timelimit = 0.0; + +static double currentTime(void){ + double t; + static sqlite3_vfs *pTimelimitVfs = 0; + if( pTimelimitVfs==0 ) pTimelimitVfs = sqlite3_vfs_find(0); + if( pTimelimitVfs->iVersion>=2 && pTimelimitVfs->xCurrentTimeInt64!=0 ){ + sqlite3_int64 tm; + pTimelimitVfs->xCurrentTimeInt64(pTimelimitVfs, &tm); + t = tm/86400000.0; + }else{ + pTimelimitVfs->xCurrentTime(pTimelimitVfs, &t); + } + return t; +} + +static void setstoptime_x( + Error *pErr, /* IN/OUT: Error code */ + int nMs /* Milliseconds until "stop time" */ +){ + if( pErr->rc==SQLITE_OK ){ + double t = currentTime(); + timelimit = t + ((double)nMs)/(1000.0*60.0*60.0*24.0); + } +} + +static int timetostop_x( + Error *pErr /* IN/OUT: Error code */ +){ + int ret = 1; + if( pErr->rc==SQLITE_OK ){ + double t = currentTime(); + ret = (t >= timelimit); + } + return ret; +} + From 820fc4933ba9ad6afff0adfbc88f7458689e317d Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 7 May 2016 18:02:53 +0000 Subject: [PATCH 032/179] Add options to bc_test1.c to make it more flexible. FossilOrigin-Name: ec6ef5f2c2dc18e1a19c205f365f4071f0870b68 --- manifest | 14 +++--- manifest.uuid | 2 +- test/bc_test1.c | 113 +++++++++++++++++++----------------------- test/tt3_core.c | 129 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 188 insertions(+), 70 deletions(-) diff --git a/manifest b/manifest index 161feb3e40..9b43e47d70 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest/bc_test1.c,\sfor\stesting\sthe\sdegree\sof\sconcurrency\sprovided\sby\sthis\sbranch\sunder\svarious\sconditions. -D 2016-05-06T21:04:47.598 +C Add\soptions\sto\sbc_test1.c\sto\smake\sit\smore\sflexible. +D 2016-05-07T18:02:53.432 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -521,7 +521,7 @@ F test/backup_malloc.test 7162d604ec2b4683c4b3799a48657fb8b5e2d450 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f F test/badutf2.test f5bc7f2d280670ecd79b9cf4f0f1760c607fe51f F test/bc_common.tcl b5e42d80305be95697e6370e015af571e5333a1c -F test/bc_test1.c d67cb1a933ae3a9116c8dc6da31e9ebf1c06ac23 +F test/bc_test1.c 3f10831d71c7e0bc53640c1989d31ded030a0fcc F test/bestindex1.test 0cf1bd2d7b97d3a3a8c10736125274f64765c4ee F test/bestindex2.test 4a06b8922ab2fd09434870da8d1cdf525aaf7060 F test/between.test 34d375fb5ce1ae283ffe82b6b233e9f38e84fc6c @@ -1295,7 +1295,7 @@ F test/triggerC.test 302d8995f5ffe63bbc15053abb3ef7a39cf5a092 F test/triggerD.test 8e7f3921a92a5797d472732108109e44575fa650 F test/triggerE.test 15fa63f1097db1f83dd62d121616006978063d1f F test/tt3_checkpoint.c 9e75cf7c1c364f52e1c47fd0f14c4340a9db0fe1 -F test/tt3_core.c 13c19feb4fe127bc86ae86d2c1237eb71c73f2ab +F test/tt3_core.c 77d7acbe43f78802284da8ef04650f6f5f32f7a0 F test/tt3_index.c 39eec10a35f57672225be4d182862152896dee4a F test/tt3_lookaside1.c 0377e202c3c2a50d688cb65ba203afeda6fafeb9 F test/tt3_stress.c c57d804716165811d979d4a719e05baccd79277f @@ -1491,7 +1491,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 91e5c07eaf884d3598df8eb54f0910a80df48397 -R 4bab1825e8924fd8c0c1d12f2130ddfc +P 128c7eaed5479b615d75ebce1d781ea661e0fb2d +R 86fffa805f6f27d07092161192388b60 U dan -Z 8e9093e5b020040f9bd55508cf282679 +Z aab288310f3155f4f7b4c6e3e47a2d89 diff --git a/manifest.uuid b/manifest.uuid index 86543cee78..c99acbf3cf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -128c7eaed5479b615d75ebce1d781ea661e0fb2d \ No newline at end of file +ec6ef5f2c2dc18e1a19c205f365f4071f0870b68 \ No newline at end of file diff --git a/test/bc_test1.c b/test/bc_test1.c index dd8ab30440..bb41e9d4ed 100644 --- a/test/bc_test1.c +++ b/test/bc_test1.c @@ -1,24 +1,35 @@ +/* +** 2016-05-07 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ #include - #include - +#include #include "tt3_core.c" typedef struct Config Config; - struct Config { int nIPT; /* --inserts-per-transaction */ int nThread; /* --threads */ int nSecond; /* --seconds */ int bMutex; /* --mutex */ - + int nAutoCkpt; /* --autockpt */ int bRm; /* --rm */ sqlite3_mutex *pMutex; }; + static char *thread_main(int iTid, void *pArg){ Config *pConfig = (Config*)pArg; Error err = {0}; /* Error code and message */ @@ -29,7 +40,10 @@ static char *thread_main(int iTid, void *pArg){ opendb(&err, &db, "xyz.db", 0); sqlite3_busy_handler(db.db, 0, 0); - execsql(&err, &db, "PRAGMA wal_autocheckpoint = 0"); + sql_script_printf(&err, &db, + "PRAGMA wal_autocheckpoint = %d;" + "PRAGMA synchronous = 0;", pConfig->nAutoCkpt + ); while( !timetostop(&err) ){ execsql(&err, &db, "BEGIN CONCURRENT"); @@ -55,67 +69,35 @@ static char *thread_main(int iTid, void *pArg){ return sqlite3_mprintf("%d/%d successful commits", nCommit, nAttempt); } -static void usage(char *zName){ - fprintf(stderr, "Usage: %s ?SWITCHES?\n", zName); - fprintf(stderr, "\n"); - fprintf(stderr, "where switches are\n"); - fprintf(stderr, " --seconds N\n"); - fprintf(stderr, " --inserts N\n"); - fprintf(stderr, " --threads N\n"); - fprintf(stderr, " --rm BOOL\n"); - fprintf(stderr, " --mutex BOOL\n"); - fprintf(stderr, "\n"); - exit(-1); -} - -int main(int argc, char **argv){ +int main(int argc, const char **argv){ Error err = {0}; /* Error code and message */ Sqlite db = {0}; /* SQLite database connection */ Threadset threads = {0}; /* Test threads */ - Config sConfig = {5, 3, 5}; + Config conf = {5, 3, 5}; int i; - for(i=1; i=3 && 0==sqlite3_strnicmp(z, "--seconds", n) ){ - if( (++i)==argc ) usage(argv[0]); - sConfig.nSecond = atoi(argv[i]); - } - - else if( n>=3 && 0==sqlite3_strnicmp(z, "--inserts", n) ){ - if( (++i)==argc ) usage(argv[0]); - sConfig.nIPT = atoi(argv[i]); - } - - else if( n>=3 && 0==sqlite3_strnicmp(z, "--threads", n) ){ - if( (++i)==argc ) usage(argv[0]); - sConfig.nThread = atoi(argv[i]); - } - - else if( n>=3 && 0==sqlite3_strnicmp(z, "--rm", n) ){ - if( (++i)==argc ) usage(argv[0]); - sConfig.bRm = atoi(argv[i]); - } - - else if( n>=3 && 0==sqlite3_strnicmp(z, "--mutex", n) ){ - if( (++i)==argc ) usage(argv[0]); - sConfig.bMutex = atoi(argv[i]); - } - - else usage(argv[0]); + CmdlineArg apArg[] = { + { "--seconds", CMDLINE_INT, offsetof(Config, nSecond) }, + { "--inserts", CMDLINE_INT, offsetof(Config, nIPT) }, + { "--threads", CMDLINE_INT, offsetof(Config, nThread) }, + { "--mutex", CMDLINE_BOOL, offsetof(Config, bMutex) }, + { "--rm", CMDLINE_BOOL, offsetof(Config, bRm) }, + { "--autockpt",CMDLINE_INT, offsetof(Config, nAutoCkpt) }, + { 0, 0, 0 } + }; + + cmdline_process(apArg, argc, argv, (void*)&conf); + if( err.rc==SQLITE_OK ){ + char *z = cmdline_construct(apArg, (void*)&conf); + printf("With: %s\n", z); + sqlite3_free(z); } - printf("With: --threads %d --inserts %d --seconds %d --rm %d --mutex %d\n", - sConfig.nThread, sConfig.nIPT, sConfig.nSecond, sConfig.bRm, - sConfig.bMutex - ); - /* Ensure the schema has been created */ - if( sConfig.bMutex ){ - sConfig.pMutex = sqlite3_mutex_alloc(SQLITE_MUTEX_RECURSIVE); + if( conf.bMutex ){ + conf.pMutex = sqlite3_mutex_alloc(SQLITE_MUTEX_RECURSIVE); } - opendb(&err, &db, "xyz.db", sConfig.bRm); + opendb(&err, &db, "xyz.db", conf.bRm); sql_script(&err, &db, "PRAGMA journal_mode = wal;" @@ -123,15 +105,22 @@ int main(int argc, char **argv){ "CREATE INDEX IF NOT EXISTS t1b ON t1(b);" "CREATE INDEX IF NOT EXISTS t1c ON t1(c);" ); - closedb(&err, &db); - setstoptime(&err, sConfig.nSecond*1000); - for(i=0; ieType ){ + case CMDLINE_INT: { + zRet = sqlite3_mprintf("%z%s%s %d", zRet, zSpace, pArg->zSwitch, + *(int*)(p + pArg->iOffset) + ); + break; + }; + + case CMDLINE_BOOL: + if( *(int*)(p + pArg->iOffset) ){ + zRet = sqlite3_mprintf("%z%s%s", zRet, zSpace, pArg->zSwitch); + } + break; + + default: + zRet = sqlite3_mprintf("%z%s%s ???", zRet, zSpace, pArg->zSwitch); + } + } + + return zRet; +} + +static void cmdline_process( + CmdlineArg *apArg, + int argc, + const char **argv, + void *pObj +){ + int i; + int iArg; + unsigned char *p = (unsigned char*)pObj; + + for(i=1; i=0 ){ + cmdline_error("ambiguous switch: %s", z); + } + iOpt = iArg; + switch( apArg[iArg].eType ){ + case CMDLINE_INT: + i++; + if( i==argc ){ + cmdline_error("option requires an argument: %s", z); + } + *(int*)(p + apArg[iArg].iOffset) = atoi(argv[i]); + break; + + case CMDLINE_BOOL: + *(int*)(p + apArg[iArg].iOffset) = 1; + break; + + default: + assert( 0 ); + cmdline_error("internal error"); + return; + } + } + } + + if( iOpt<0 ){ + cmdline_usage(argv[0], apArg); + } + } +} + +/* +** End of command line processing utilities. +*************************************************************************/ + + /* * This code implements the MD5 message-digest algorithm. * The algorithm is due to Ron Rivest. This code was From 12891cc492331a57a5c602951ccbee94a532e034 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 21 May 2016 18:50:56 +0000 Subject: [PATCH 033/179] Use pthreads mutexes and conditions to synchronize threads in bc_test1. FossilOrigin-Name: f33aa76f074d8686a5a5c0edecabb71cb259c48d --- manifest | 12 +- manifest.uuid | 2 +- test/bc_test1.c | 353 ++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 349 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 9b43e47d70..8db6dfd331 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\soptions\sto\sbc_test1.c\sto\smake\sit\smore\sflexible. -D 2016-05-07T18:02:53.432 +C Use\spthreads\smutexes\sand\sconditions\sto\ssynchronize\sthreads\sin\sbc_test1. +D 2016-05-21T18:50:56.299 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -521,7 +521,7 @@ F test/backup_malloc.test 7162d604ec2b4683c4b3799a48657fb8b5e2d450 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f F test/badutf2.test f5bc7f2d280670ecd79b9cf4f0f1760c607fe51f F test/bc_common.tcl b5e42d80305be95697e6370e015af571e5333a1c -F test/bc_test1.c 3f10831d71c7e0bc53640c1989d31ded030a0fcc +F test/bc_test1.c 874d03ffabbd565a6cf9a7eec23d3deafd3c8f1c F test/bestindex1.test 0cf1bd2d7b97d3a3a8c10736125274f64765c4ee F test/bestindex2.test 4a06b8922ab2fd09434870da8d1cdf525aaf7060 F test/between.test 34d375fb5ce1ae283ffe82b6b233e9f38e84fc6c @@ -1491,7 +1491,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 128c7eaed5479b615d75ebce1d781ea661e0fb2d -R 86fffa805f6f27d07092161192388b60 +P ec6ef5f2c2dc18e1a19c205f365f4071f0870b68 +R 9d74e8a08d57f3a5262646881b7fcf49 U dan -Z aab288310f3155f4f7b4c6e3e47a2d89 +Z 51f0c57ee889307d403d3ab523c9aec8 diff --git a/manifest.uuid b/manifest.uuid index c99acbf3cf..2e3f1b7111 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec6ef5f2c2dc18e1a19c205f365f4071f0870b68 \ No newline at end of file +f33aa76f074d8686a5a5c0edecabb71cb259c48d \ No newline at end of file diff --git a/test/bc_test1.c b/test/bc_test1.c index bb41e9d4ed..290e04067b 100644 --- a/test/bc_test1.c +++ b/test/bc_test1.c @@ -26,11 +26,326 @@ struct Config { int bMutex; /* --mutex */ int nAutoCkpt; /* --autockpt */ int bRm; /* --rm */ - sqlite3_mutex *pMutex; + + pthread_cond_t cond; + pthread_mutex_t mutex; + int nCondWait; /* Number of threads waiting on hCond */ + sqlite3_vfs *pVfs; +}; + +typedef struct WalHookCtx WalHookCtx; +struct WalHookCtx { + Config *pConfig; + Sqlite *pDb; + Error *pErr; +}; + +typedef struct VfsWrapperFd VfsWrapperFd; +struct VfsWrapperFd { + sqlite3_file base; /* Base class */ + int bWriter; /* True if holding shm WRITER lock */ + Config *pConfig; + sqlite3_file *pFd; /* Underlying file descriptor */ }; +/* Methods of the wrapper VFS */ +static int vfsWrapOpen(sqlite3_vfs*, const char*, sqlite3_file*, int, int*); +static int vfsWrapDelete(sqlite3_vfs*, const char*, int); +static int vfsWrapAccess(sqlite3_vfs*, const char*, int, int*); +static int vfsWrapFullPathname(sqlite3_vfs*, const char *, int, char*); +static void *vfsWrapDlOpen(sqlite3_vfs*, const char*); +static void vfsWrapDlError(sqlite3_vfs*, int, char*); +static void (*vfsWrapDlSym(sqlite3_vfs*,void*, const char*))(void); +static void vfsWrapDlClose(sqlite3_vfs*, void*); +static int vfsWrapRandomness(sqlite3_vfs*, int, char*); +static int vfsWrapSleep(sqlite3_vfs*, int); +static int vfsWrapCurrentTime(sqlite3_vfs*, double*); +static int vfsWrapGetLastError(sqlite3_vfs*, int, char*); +static int vfsWrapCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); +static int vfsWrapSetSystemCall(sqlite3_vfs*, const char*, sqlite3_syscall_ptr); +static sqlite3_syscall_ptr vfsWrapGetSystemCall(sqlite3_vfs*, const char*); +static const char *vfsWrapNextSystemCall(sqlite3_vfs*, const char*); + +/* Methods of wrapper sqlite3_io_methods object (see vfsWrapOpen()) */ +static int vfsWrapClose(sqlite3_file*); +static int vfsWrapRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int vfsWrapWrite(sqlite3_file*, const void*, int iAmt, sqlite3_int64); +static int vfsWrapTruncate(sqlite3_file*, sqlite3_int64 size); +static int vfsWrapSync(sqlite3_file*, int flags); +static int vfsWrapFileSize(sqlite3_file*, sqlite3_int64 *pSize); +static int vfsWrapLock(sqlite3_file*, int); +static int vfsWrapUnlock(sqlite3_file*, int); +static int vfsWrapCheckReservedLock(sqlite3_file*, int *pResOut); +static int vfsWrapFileControl(sqlite3_file*, int op, void *pArg); +static int vfsWrapSectorSize(sqlite3_file*); +static int vfsWrapDeviceCharacteristics(sqlite3_file*); +static int vfsWrapShmMap(sqlite3_file*, int iPg, int, int, void volatile**); +static int vfsWrapShmLock(sqlite3_file*, int offset, int n, int flags); +static void vfsWrapShmBarrier(sqlite3_file*); +static int vfsWrapShmUnmap(sqlite3_file*, int deleteFlag); +static int vfsWrapFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **); +static int vfsWrapUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); + +static int vfsWrapOpen( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_file *pFd, + int flags, + int *fout +){ + static sqlite3_io_methods methods = { + 3, + vfsWrapClose, vfsWrapRead, vfsWrapWrite, + vfsWrapTruncate, vfsWrapSync, vfsWrapFileSize, + vfsWrapLock, vfsWrapUnlock, vfsWrapCheckReservedLock, + vfsWrapFileControl, vfsWrapSectorSize, vfsWrapDeviceCharacteristics, + vfsWrapShmMap, vfsWrapShmLock, vfsWrapShmBarrier, + vfsWrapShmUnmap, vfsWrapFetch, vfsWrapUnfetch + }; + + Config *pConfig = (Config*)pVfs->pAppData; + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + int rc; + + pWrapper->pFd = (sqlite3_file*)&pWrapper[1]; + pWrapper->pConfig = pConfig; + rc = pConfig->pVfs->xOpen(pConfig->pVfs, zName, pWrapper->pFd, flags, fout); + if( rc==SQLITE_OK ){ + pWrapper->base.pMethods = &methods; + } + return rc; +} + +static int vfsWrapDelete(sqlite3_vfs *pVfs, const char *a, int b){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xDelete(pConfig->pVfs, a, b); +} +static int vfsWrapAccess(sqlite3_vfs *pVfs, const char *a, int b, int *c){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xAccess(pConfig->pVfs, a, b, c); +} +static int vfsWrapFullPathname(sqlite3_vfs *pVfs, const char *a, int b, char*c){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xFullPathname(pConfig->pVfs, a, b, c); +} +static void *vfsWrapDlOpen(sqlite3_vfs *pVfs, const char *a){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xDlOpen(pConfig->pVfs, a); +} +static void vfsWrapDlError(sqlite3_vfs *pVfs, int a, char *b){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xDlError(pConfig->pVfs, a, b); +} +static void (*vfsWrapDlSym(sqlite3_vfs *pVfs, void *a, const char *b))(void){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xDlSym(pConfig->pVfs, a, b); +} +static void vfsWrapDlClose(sqlite3_vfs *pVfs, void *a){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xDlClose(pConfig->pVfs, a); +} +static int vfsWrapRandomness(sqlite3_vfs *pVfs, int a, char *b){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xRandomness(pConfig->pVfs, a, b); +} +static int vfsWrapSleep(sqlite3_vfs *pVfs, int a){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xSleep(pConfig->pVfs, a); +} +static int vfsWrapCurrentTime(sqlite3_vfs *pVfs, double *a){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xCurrentTime(pConfig->pVfs, a); +} +static int vfsWrapGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xGetLastError(pConfig->pVfs, a, b); +} +static int vfsWrapCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *a){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xCurrentTimeInt64(pConfig->pVfs, a); +} +static int vfsWrapSetSystemCall( + sqlite3_vfs *pVfs, + const char *a, + sqlite3_syscall_ptr b +){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xSetSystemCall(pConfig->pVfs, a, b); +} +static sqlite3_syscall_ptr vfsWrapGetSystemCall( + sqlite3_vfs *pVfs, + const char *a +){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xGetSystemCall(pConfig->pVfs, a); +} +static const char *vfsWrapNextSystemCall(sqlite3_vfs *pVfs, const char *a){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xNextSystemCall(pConfig->pVfs, a); +} + +static int vfsWrapClose(sqlite3_file *pFd){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + pWrapper->pFd->pMethods->xClose(pWrapper->pFd); + pWrapper->pFd = 0; + return SQLITE_OK; +} +static int vfsWrapRead(sqlite3_file *pFd, void *a, int b, sqlite3_int64 c){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xRead(pWrapper->pFd, a, b, c); +} +static int vfsWrapWrite( + sqlite3_file *pFd, + const void *a, int b, + sqlite3_int64 c +){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xWrite(pWrapper->pFd, a, b, c); +} +static int vfsWrapTruncate(sqlite3_file *pFd, sqlite3_int64 a){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xTruncate(pWrapper->pFd, a); +} +static int vfsWrapSync(sqlite3_file *pFd, int a){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xSync(pWrapper->pFd, a); +} +static int vfsWrapFileSize(sqlite3_file *pFd, sqlite3_int64 *a){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xFileSize(pWrapper->pFd, a); +} +static int vfsWrapLock(sqlite3_file *pFd, int a){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xLock(pWrapper->pFd, a); +} +static int vfsWrapUnlock(sqlite3_file *pFd, int a){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xUnlock(pWrapper->pFd, a); +} +static int vfsWrapCheckReservedLock(sqlite3_file *pFd, int *a){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xCheckReservedLock(pWrapper->pFd, a); +} +static int vfsWrapFileControl(sqlite3_file *pFd, int a, void *b){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xFileControl(pWrapper->pFd, a, b); +} +static int vfsWrapSectorSize(sqlite3_file *pFd){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xSectorSize(pWrapper->pFd); +} +static int vfsWrapDeviceCharacteristics(sqlite3_file *pFd){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xDeviceCharacteristics(pWrapper->pFd); +} +static int vfsWrapShmMap( + sqlite3_file *pFd, + int a, int b, int c, + void volatile **d +){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xShmMap(pWrapper->pFd, a, b, c, d); +} +static int vfsWrapShmLock(sqlite3_file *pFd, int offset, int n, int flags){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + Config *pConfig = pWrapper->pConfig; + int bMutex = 0; + int rc; + + if( (offset==0 && n==1) + && (flags & SQLITE_SHM_LOCK) && (flags & SQLITE_SHM_EXCLUSIVE) + ){ + pthread_mutex_lock(&pConfig->mutex); + pWrapper->bWriter = 1; + bMutex = 1; + } + + if( offset==0 && (flags & SQLITE_SHM_UNLOCK) && pWrapper->bWriter ){ + pthread_mutex_unlock(&pConfig->mutex); + pWrapper->bWriter = 0; + } + + rc = pWrapper->pFd->pMethods->xShmLock(pWrapper->pFd, offset, n, flags); + + if( rc!=SQLITE_OK && bMutex ){ + pthread_mutex_unlock(&pConfig->mutex); + pWrapper->bWriter = 0; + } + + return rc; +} +static void vfsWrapShmBarrier(sqlite3_file *pFd){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xShmBarrier(pWrapper->pFd); +} +static int vfsWrapShmUnmap(sqlite3_file *pFd, int a){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xShmUnmap(pWrapper->pFd, a); +} +static int vfsWrapFetch(sqlite3_file *pFd, sqlite3_int64 a, int b, void **c){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xFetch(pWrapper->pFd, a, b, c); +} +static int vfsWrapUnfetch(sqlite3_file *pFd, sqlite3_int64 a, void *b){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xUnfetch(pWrapper->pFd, a, b); +} + +static void create_vfs(Config *pConfig){ + static sqlite3_vfs vfs = { + 3, 0, 0, 0, "wrapper", 0, + vfsWrapOpen, vfsWrapDelete, vfsWrapAccess, + vfsWrapFullPathname, vfsWrapDlOpen, vfsWrapDlError, + vfsWrapDlSym, vfsWrapDlClose, vfsWrapRandomness, + vfsWrapSleep, vfsWrapCurrentTime, vfsWrapGetLastError, + vfsWrapCurrentTimeInt64, vfsWrapSetSystemCall, vfsWrapGetSystemCall, + vfsWrapNextSystemCall + }; + sqlite3_vfs *pVfs; + + pVfs = sqlite3_vfs_find(0); + vfs.mxPathname = pVfs->mxPathname; + vfs.szOsFile = pVfs->szOsFile + sizeof(VfsWrapperFd); + vfs.pAppData = (void*)pConfig; + pConfig->pVfs = pVfs; + + sqlite3_vfs_register(&vfs, 1); +} + + +/* +** Wal hook used by connections in thread_main(). +*/ +static int thread_wal_hook( + void *pArg, /* Pointer to Config object */ + sqlite3 *db, + const char *zDb, + int nFrame +){ + WalHookCtx *pCtx = (WalHookCtx*)pArg; + Config *pConfig = pCtx->pConfig; + + if( nFrame>=pConfig->nAutoCkpt ){ + pthread_mutex_lock(&pConfig->mutex); + if( pConfig->nCondWait>=0 ){ + pConfig->nCondWait++; + if( pConfig->nCondWait==pConfig->nThread ){ + execsql(pCtx->pErr, pCtx->pDb, "PRAGMA wal_checkpoint"); + pthread_cond_broadcast(&pConfig->cond); + }else{ + pthread_cond_wait(&pConfig->cond, &pConfig->mutex); + } + pConfig->nCondWait--; + } + pthread_mutex_unlock(&pConfig->mutex); + } + + return SQLITE_OK; +} + static char *thread_main(int iTid, void *pArg){ + WalHookCtx ctx; Config *pConfig = (Config*)pArg; Error err = {0}; /* Error code and message */ Sqlite db = {0}; /* SQLite database connection */ @@ -45,6 +360,11 @@ static char *thread_main(int iTid, void *pArg){ "PRAGMA synchronous = 0;", pConfig->nAutoCkpt ); + ctx.pConfig = pConfig; + ctx.pErr = &err; + ctx.pDb = &db; + sqlite3_wal_hook(db.db, thread_wal_hook, (void*)&ctx); + while( !timetostop(&err) ){ execsql(&err, &db, "BEGIN CONCURRENT"); for(j=0; jnIPT; j++){ @@ -53,9 +373,7 @@ static char *thread_main(int iTid, void *pArg){ "(randomblob(10), randomblob(20), randomblob(30), randomblob(200))" ); } - sqlite3_mutex_enter(pConfig->pMutex); execsql(&err, &db, "COMMIT"); - sqlite3_mutex_leave(pConfig->pMutex); nAttempt++; if( err.rc==SQLITE_OK ){ nCommit++; @@ -66,6 +384,12 @@ static char *thread_main(int iTid, void *pArg){ } closedb(&err, &db); + + pthread_mutex_lock(&pConfig->mutex); + pConfig->nCondWait = -1; + pthread_cond_broadcast(&pConfig->cond); + pthread_mutex_unlock(&pConfig->mutex); + return sqlite3_mprintf("%d/%d successful commits", nCommit, nAttempt); } @@ -93,12 +417,14 @@ int main(int argc, const char **argv){ sqlite3_free(z); } + /* Create the special VFS - "wrapper". And the mutex and condition + ** variable. */ + create_vfs(&conf); + pthread_mutex_init(&conf.mutex, 0); + pthread_cond_init(&conf.cond, 0); + /* Ensure the schema has been created */ - if( conf.bMutex ){ - conf.pMutex = sqlite3_mutex_alloc(SQLITE_MUTEX_RECURSIVE); - } opendb(&err, &db, "xyz.db", conf.bRm); - sql_script(&err, &db, "PRAGMA journal_mode = wal;" "CREATE TABLE IF NOT EXISTS t1(a PRIMARY KEY, b, c, d) WITHOUT ROWID;" @@ -107,10 +433,16 @@ int main(int argc, const char **argv){ ); setstoptime(&err, conf.nSecond*1000); - for(i=0; i Date: Thu, 26 May 2016 20:52:15 +0000 Subject: [PATCH 034/179] Add further instrumentation to the bc_test1.c test app. FossilOrigin-Name: 5528de4a53c19557798b6169e1d738f1a301e131 --- manifest | 14 ++--- manifest.uuid | 2 +- test/bc_test1.c | 154 +++++++++++++++++++++++++++++++++++++++--------- test/tt3_core.c | 22 ++++++- 4 files changed, 154 insertions(+), 38 deletions(-) diff --git a/manifest b/manifest index 8db6dfd331..1c2a37b300 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\spthreads\smutexes\sand\sconditions\sto\ssynchronize\sthreads\sin\sbc_test1. -D 2016-05-21T18:50:56.299 +C Add\sfurther\sinstrumentation\sto\sthe\sbc_test1.c\stest\sapp. +D 2016-05-26T20:52:15.750 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -521,7 +521,7 @@ F test/backup_malloc.test 7162d604ec2b4683c4b3799a48657fb8b5e2d450 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f F test/badutf2.test f5bc7f2d280670ecd79b9cf4f0f1760c607fe51f F test/bc_common.tcl b5e42d80305be95697e6370e015af571e5333a1c -F test/bc_test1.c 874d03ffabbd565a6cf9a7eec23d3deafd3c8f1c +F test/bc_test1.c 05cd99276a5dc2de0e28f3b1095c343393e12f87 F test/bestindex1.test 0cf1bd2d7b97d3a3a8c10736125274f64765c4ee F test/bestindex2.test 4a06b8922ab2fd09434870da8d1cdf525aaf7060 F test/between.test 34d375fb5ce1ae283ffe82b6b233e9f38e84fc6c @@ -1295,7 +1295,7 @@ F test/triggerC.test 302d8995f5ffe63bbc15053abb3ef7a39cf5a092 F test/triggerD.test 8e7f3921a92a5797d472732108109e44575fa650 F test/triggerE.test 15fa63f1097db1f83dd62d121616006978063d1f F test/tt3_checkpoint.c 9e75cf7c1c364f52e1c47fd0f14c4340a9db0fe1 -F test/tt3_core.c 77d7acbe43f78802284da8ef04650f6f5f32f7a0 +F test/tt3_core.c 01a7cd1a4f10a666f404c46360da15468522d52e F test/tt3_index.c 39eec10a35f57672225be4d182862152896dee4a F test/tt3_lookaside1.c 0377e202c3c2a50d688cb65ba203afeda6fafeb9 F test/tt3_stress.c c57d804716165811d979d4a719e05baccd79277f @@ -1491,7 +1491,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ec6ef5f2c2dc18e1a19c205f365f4071f0870b68 -R 9d74e8a08d57f3a5262646881b7fcf49 +P f33aa76f074d8686a5a5c0edecabb71cb259c48d +R 3e6ef002903d2cc9312cf24b4caaa4ad U dan -Z 51f0c57ee889307d403d3ab523c9aec8 +Z 14c81df2e8a28af21f0d7f0b336be7a7 diff --git a/manifest.uuid b/manifest.uuid index 2e3f1b7111..b3462a0dfc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f33aa76f074d8686a5a5c0edecabb71cb259c48d \ No newline at end of file +5528de4a53c19557798b6169e1d738f1a301e131 \ No newline at end of file diff --git a/test/bc_test1.c b/test/bc_test1.c index 290e04067b..18b623bd1d 100644 --- a/test/bc_test1.c +++ b/test/bc_test1.c @@ -17,8 +17,28 @@ #include #include "tt3_core.c" +#ifdef USE_OSINST +# include "../src/test_osinst.c" +#else +# define vfslog_time() 0 +#endif typedef struct Config Config; +typedef struct ThreadCtx ThreadCtx; + +#define THREAD_TIME_INSERT 0 +#define THREAD_TIME_COMMIT 1 +#define THREAD_TIME_ROLLBACK 2 +#define THREAD_TIME_WRITER 3 +#define THREAD_TIME_CKPT 4 + +struct ThreadCtx { + Config *pConfig; + Sqlite *pDb; + Error *pErr; + sqlite3_int64 aTime[5]; +}; + struct Config { int nIPT; /* --inserts-per-transaction */ int nThread; /* --threads */ @@ -26,6 +46,12 @@ struct Config { int bMutex; /* --mutex */ int nAutoCkpt; /* --autockpt */ int bRm; /* --rm */ + int bClearCache; /* --clear-cache */ + int nMmap; /* mmap limit in MB */ + char *zFile; + int bOsinst; /* True to use osinst */ + + ThreadCtx *aCtx; /* Array of size nThread */ pthread_cond_t cond; pthread_mutex_t mutex; @@ -33,17 +59,12 @@ struct Config { sqlite3_vfs *pVfs; }; -typedef struct WalHookCtx WalHookCtx; -struct WalHookCtx { - Config *pConfig; - Sqlite *pDb; - Error *pErr; -}; typedef struct VfsWrapperFd VfsWrapperFd; struct VfsWrapperFd { sqlite3_file base; /* Base class */ int bWriter; /* True if holding shm WRITER lock */ + int iTid; Config *pConfig; sqlite3_file *pFd; /* Underlying file descriptor */ }; @@ -107,6 +128,11 @@ static int vfsWrapOpen( VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; int rc; + memset(pWrapper, 0, sizeof(VfsWrapperFd)); + if( flags & SQLITE_OPEN_MAIN_DB ){ + pWrapper->iTid = (int)sqlite3_uri_int64(zName, "tid", 0); + } + pWrapper->pFd = (sqlite3_file*)&pWrapper[1]; pWrapper->pConfig = pConfig; rc = pConfig->pVfs->xOpen(pConfig->pVfs, zName, pWrapper->pFd, flags, fout); @@ -258,18 +284,24 @@ static int vfsWrapShmLock(sqlite3_file *pFd, int offset, int n, int flags){ pthread_mutex_lock(&pConfig->mutex); pWrapper->bWriter = 1; bMutex = 1; - } - - if( offset==0 && (flags & SQLITE_SHM_UNLOCK) && pWrapper->bWriter ){ - pthread_mutex_unlock(&pConfig->mutex); - pWrapper->bWriter = 0; + if( pWrapper->iTid ){ + sqlite3_int64 t = vfslog_time(); + pConfig->aCtx[pWrapper->iTid-1].aTime[THREAD_TIME_WRITER] -= t; + } } rc = pWrapper->pFd->pMethods->xShmLock(pWrapper->pFd, offset, n, flags); - if( rc!=SQLITE_OK && bMutex ){ + if( (rc!=SQLITE_OK && bMutex) + || (offset==0 && (flags & SQLITE_SHM_UNLOCK) && pWrapper->bWriter) + ){ + assert( pWrapper->bWriter ); pthread_mutex_unlock(&pConfig->mutex); pWrapper->bWriter = 0; + if( pWrapper->iTid ){ + sqlite3_int64 t = vfslog_time(); + pConfig->aCtx[pWrapper->iTid-1].aTime[THREAD_TIME_WRITER] += t; + } } return rc; @@ -317,15 +349,16 @@ static void create_vfs(Config *pConfig){ ** Wal hook used by connections in thread_main(). */ static int thread_wal_hook( - void *pArg, /* Pointer to Config object */ + void *pArg, /* Pointer to ThreadCtx object */ sqlite3 *db, const char *zDb, int nFrame ){ - WalHookCtx *pCtx = (WalHookCtx*)pArg; + ThreadCtx *pCtx = (ThreadCtx*)pArg; Config *pConfig = pCtx->pConfig; - if( nFrame>=pConfig->nAutoCkpt ){ + if( pConfig->nAutoCkpt && nFrame>=pConfig->nAutoCkpt ){ + pCtx->aTime[THREAD_TIME_CKPT] -= vfslog_time(); pthread_mutex_lock(&pConfig->mutex); if( pConfig->nCondWait>=0 ){ pConfig->nCondWait++; @@ -338,6 +371,7 @@ static int thread_wal_hook( pConfig->nCondWait--; } pthread_mutex_unlock(&pConfig->mutex); + pCtx->aTime[THREAD_TIME_CKPT] += vfslog_time(); } return SQLITE_OK; @@ -345,35 +379,63 @@ static int thread_wal_hook( static char *thread_main(int iTid, void *pArg){ - WalHookCtx ctx; Config *pConfig = (Config*)pArg; Error err = {0}; /* Error code and message */ Sqlite db = {0}; /* SQLite database connection */ int nAttempt = 0; /* Attempted transactions */ int nCommit = 0; /* Successful transactions */ int j; + ThreadCtx *pCtx = &pConfig->aCtx[iTid-1]; + char *zUri = 0; + +#ifdef USE_OSINST + char *zOsinstName = 0; + char *zLogName = 0; + if( pConfig->bOsinst ){ + zOsinstName = sqlite3_mprintf("osinst%d", iTid); + zLogName = sqlite3_mprintf("bc_test1.log.%d.%d", (int)getpid(), iTid); + zUri = sqlite3_mprintf( + "file:%s?vfs=%s&tid=%d", pConfig->zFile, zOsinstName, iTid + ); + sqlite3_vfslog_new(zOsinstName, 0, zLogName); + opendb(&err, &db, zUri, 0); + }else +#endif + { + zUri = sqlite3_mprintf("file:%s?tid=%d", pConfig->zFile, iTid); + opendb(&err, &db, zUri, 0); + } - opendb(&err, &db, "xyz.db", 0); sqlite3_busy_handler(db.db, 0, 0); sql_script_printf(&err, &db, - "PRAGMA wal_autocheckpoint = %d;" - "PRAGMA synchronous = 0;", pConfig->nAutoCkpt + "PRAGMA wal_autocheckpoint = 0;" + "PRAGMA synchronous = 0;" + "PRAGMA mmap_limit = %lld;", + (i64)(pConfig->nMmap) * 1024 * 1024 ); - ctx.pConfig = pConfig; - ctx.pErr = &err; - ctx.pDb = &db; - sqlite3_wal_hook(db.db, thread_wal_hook, (void*)&ctx); + pCtx->pConfig = pConfig; + pCtx->pErr = &err; + pCtx->pDb = &db; + sqlite3_wal_hook(db.db, thread_wal_hook, (void*)pCtx); while( !timetostop(&err) ){ execsql(&err, &db, "BEGIN CONCURRENT"); + + pCtx->aTime[THREAD_TIME_INSERT] -= vfslog_time(); for(j=0; jnIPT; j++){ execsql(&err, &db, "INSERT INTO t1 VALUES" "(randomblob(10), randomblob(20), randomblob(30), randomblob(200))" ); } + pCtx->aTime[THREAD_TIME_INSERT] += vfslog_time(); + + pCtx->aTime[THREAD_TIME_COMMIT] -= vfslog_time(); execsql(&err, &db, "COMMIT"); + pCtx->aTime[THREAD_TIME_COMMIT] += vfslog_time(); + + pCtx->aTime[THREAD_TIME_ROLLBACK] -= vfslog_time(); nAttempt++; if( err.rc==SQLITE_OK ){ nCommit++; @@ -381,16 +443,41 @@ static char *thread_main(int iTid, void *pArg){ clear_error(&err, SQLITE_BUSY); execsql(&err, &db, "ROLLBACK"); } + pCtx->aTime[THREAD_TIME_ROLLBACK] += vfslog_time(); + + if( pConfig->bClearCache ){ + sqlite3_db_release_memory(db.db); + } } closedb(&err, &db); +#ifdef USE_OSINST + if( pConfig->bOsinst ){ + sqlite3_vfslog_finalize(zOsinstName); + sqlite3_free(zOsinstName); + sqlite3_free(zLogName); + } +#endif + sqlite3_free(zUri); + pthread_mutex_lock(&pConfig->mutex); pConfig->nCondWait = -1; pthread_cond_broadcast(&pConfig->cond); pthread_mutex_unlock(&pConfig->mutex); - return sqlite3_mprintf("%d/%d successful commits", nCommit, nAttempt); + return sqlite3_mprintf("commits: %d/%d insert: %dms" + " commit: %dms" + " rollback: %dms" + " writer: %dms" + " checkpoint: %dms", + nCommit, nAttempt, + (int)(pCtx->aTime[THREAD_TIME_INSERT]/1000), + (int)(pCtx->aTime[THREAD_TIME_COMMIT]/1000), + (int)(pCtx->aTime[THREAD_TIME_ROLLBACK]/1000), + (int)(pCtx->aTime[THREAD_TIME_WRITER]/1000), + (int)(pCtx->aTime[THREAD_TIME_CKPT]/1000) + ); } int main(int argc, const char **argv){ @@ -407,6 +494,10 @@ int main(int argc, const char **argv){ { "--mutex", CMDLINE_BOOL, offsetof(Config, bMutex) }, { "--rm", CMDLINE_BOOL, offsetof(Config, bRm) }, { "--autockpt",CMDLINE_INT, offsetof(Config, nAutoCkpt) }, + { "--mmap", CMDLINE_INT, offsetof(Config, nMmap) }, + { "--clear-cache", CMDLINE_BOOL, offsetof(Config, bClearCache) }, + { "--file", CMDLINE_STRING, offsetof(Config, zFile) }, + { "--osinst", CMDLINE_BOOL, offsetof(Config, bOsinst) }, { 0, 0, 0 } }; @@ -416,6 +507,9 @@ int main(int argc, const char **argv){ printf("With: %s\n", z); sqlite3_free(z); } + if( conf.zFile==0 ){ + conf.zFile = "xyz.db"; + } /* Create the special VFS - "wrapper". And the mutex and condition ** variable. */ @@ -423,8 +517,11 @@ int main(int argc, const char **argv){ pthread_mutex_init(&conf.mutex, 0); pthread_cond_init(&conf.cond, 0); + conf.aCtx = sqlite3_malloc(sizeof(ThreadCtx) * conf.nThread); + memset(conf.aCtx, 0, sizeof(ThreadCtx) * conf.nThread); + /* Ensure the schema has been created */ - opendb(&err, &db, "xyz.db", conf.bRm); + opendb(&err, &db, conf.zFile, conf.bRm); sql_script(&err, &db, "PRAGMA journal_mode = wal;" "CREATE TABLE IF NOT EXISTS t1(a PRIMARY KEY, b, c, d) WITHOUT ROWID;" @@ -434,7 +531,7 @@ int main(int argc, const char **argv){ setstoptime(&err, conf.nSecond*1000); if( conf.nThread==1 ){ - char *z = thread_main(0, (void*)&conf); + char *z = thread_main(1, (void*)&conf); printf("Thread 0 says: %s\n", (z==0 ? "..." : z)); fflush(stdout); }else{ @@ -445,10 +542,11 @@ int main(int argc, const char **argv){ } if( err.rc==SQLITE_OK ){ - printf("Database is %dK\n", (int)(filesize(&err, "xyz.db") / 1024)); + printf("Database is %dK\n", (int)(filesize(&err, conf.zFile) / 1024)); } if( err.rc==SQLITE_OK ){ - printf("Wal file is %dK\n", (int)(filesize(&err, "xyz.db-wal") / 1024)); + char *zWal = sqlite3_mprintf("%s-wal", conf.zFile); + printf("Wal file is %dK\n", (int)(filesize(&err, zWal) / 1024)); } closedb(&err, &db); diff --git a/test/tt3_core.c b/test/tt3_core.c index 65c04f0204..dffadac9c8 100644 --- a/test/tt3_core.c +++ b/test/tt3_core.c @@ -69,8 +69,9 @@ /************************************************************************ ** Start of command line processing utilities. */ -#define CMDLINE_INT 1 -#define CMDLINE_BOOL 2 +#define CMDLINE_INT 1 +#define CMDLINE_BOOL 2 +#define CMDLINE_STRING 3 typedef struct CmdlineArg CmdlineArg; struct CmdlineArg { @@ -98,6 +99,7 @@ static void cmdline_usage(const char *zPrg, CmdlineArg *apArg){ for(i=0; apArg[i].zSwitch; i++){ const char *zExtra = ""; switch( apArg[i].eType ){ + case CMDLINE_STRING: zExtra = "STRING"; break; case CMDLINE_INT: zExtra = "N"; break; case CMDLINE_BOOL: zExtra = ""; break; default: @@ -120,6 +122,14 @@ static char *cmdline_construct(CmdlineArg *apArg, void *pObj){ CmdlineArg *pArg = &apArg[iArg]; switch( pArg->eType ){ + case CMDLINE_STRING: { + char *zVal = *(char**)(p + pArg->iOffset); + if( zVal ){ + zRet = sqlite3_mprintf("%z%s%s %s", zRet, zSpace, pArg->zSwitch,zVal); + } + break; + }; + case CMDLINE_INT: { zRet = sqlite3_mprintf("%z%s%s %d", zRet, zSpace, pArg->zSwitch, *(int*)(p + pArg->iOffset) @@ -171,6 +181,14 @@ static void cmdline_process( *(int*)(p + apArg[iArg].iOffset) = atoi(argv[i]); break; + case CMDLINE_STRING: + i++; + if( i==argc ){ + cmdline_error("option requires an argument: %s", z); + } + *(char**)(p + apArg[iArg].iOffset) = sqlite3_mprintf("%s", argv[i]); + break; + case CMDLINE_BOOL: *(int*)(p + apArg[iArg].iOffset) = 1; break; From 96b9dc9b36e7911dc3f282e39d3ea3dc96642013 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 30 May 2016 05:45:32 +0000 Subject: [PATCH 035/179] Minor tweaks to the bc_test1 test program. FossilOrigin-Name: d0d0bab4e92402b6af98366be6e8955588613a51 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/bc_test1.c | 21 +++++++++++---------- test/tt3_core.c | 5 +++++ 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 1c2a37b300..9eeb419b53 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sfurther\sinstrumentation\sto\sthe\sbc_test1.c\stest\sapp. -D 2016-05-26T20:52:15.750 +C Minor\stweaks\sto\sthe\sbc_test1\stest\sprogram. +D 2016-05-30T05:45:32.809 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -521,7 +521,7 @@ F test/backup_malloc.test 7162d604ec2b4683c4b3799a48657fb8b5e2d450 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f F test/badutf2.test f5bc7f2d280670ecd79b9cf4f0f1760c607fe51f F test/bc_common.tcl b5e42d80305be95697e6370e015af571e5333a1c -F test/bc_test1.c 05cd99276a5dc2de0e28f3b1095c343393e12f87 +F test/bc_test1.c 64b3438cbb0a267de11c2c01e9ec7a96ebdbc3a3 F test/bestindex1.test 0cf1bd2d7b97d3a3a8c10736125274f64765c4ee F test/bestindex2.test 4a06b8922ab2fd09434870da8d1cdf525aaf7060 F test/between.test 34d375fb5ce1ae283ffe82b6b233e9f38e84fc6c @@ -1295,7 +1295,7 @@ F test/triggerC.test 302d8995f5ffe63bbc15053abb3ef7a39cf5a092 F test/triggerD.test 8e7f3921a92a5797d472732108109e44575fa650 F test/triggerE.test 15fa63f1097db1f83dd62d121616006978063d1f F test/tt3_checkpoint.c 9e75cf7c1c364f52e1c47fd0f14c4340a9db0fe1 -F test/tt3_core.c 01a7cd1a4f10a666f404c46360da15468522d52e +F test/tt3_core.c 8cd89ead95410f70e7fb02c79f1e040f9c5ad5cf F test/tt3_index.c 39eec10a35f57672225be4d182862152896dee4a F test/tt3_lookaside1.c 0377e202c3c2a50d688cb65ba203afeda6fafeb9 F test/tt3_stress.c c57d804716165811d979d4a719e05baccd79277f @@ -1491,7 +1491,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f33aa76f074d8686a5a5c0edecabb71cb259c48d -R 3e6ef002903d2cc9312cf24b4caaa4ad +P 5528de4a53c19557798b6169e1d738f1a301e131 +R 4e1a86c5f70b95ae38ae7d31e017c12f U dan -Z 14c81df2e8a28af21f0d7f0b336be7a7 +Z f49d8d592868742b4ebcfcd7d10075da diff --git a/manifest.uuid b/manifest.uuid index b3462a0dfc..250d42f7a8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5528de4a53c19557798b6169e1d738f1a301e131 \ No newline at end of file +d0d0bab4e92402b6af98366be6e8955588613a51 \ No newline at end of file diff --git a/test/bc_test1.c b/test/bc_test1.c index 18b623bd1d..6fbb7ca8e9 100644 --- a/test/bc_test1.c +++ b/test/bc_test1.c @@ -488,19 +488,20 @@ int main(int argc, const char **argv){ int i; CmdlineArg apArg[] = { - { "--seconds", CMDLINE_INT, offsetof(Config, nSecond) }, - { "--inserts", CMDLINE_INT, offsetof(Config, nIPT) }, - { "--threads", CMDLINE_INT, offsetof(Config, nThread) }, - { "--mutex", CMDLINE_BOOL, offsetof(Config, bMutex) }, - { "--rm", CMDLINE_BOOL, offsetof(Config, bRm) }, - { "--autockpt",CMDLINE_INT, offsetof(Config, nAutoCkpt) }, - { "--mmap", CMDLINE_INT, offsetof(Config, nMmap) }, - { "--clear-cache", CMDLINE_BOOL, offsetof(Config, bClearCache) }, - { "--file", CMDLINE_STRING, offsetof(Config, zFile) }, - { "--osinst", CMDLINE_BOOL, offsetof(Config, bOsinst) }, + { "-seconds", CMDLINE_INT, offsetof(Config, nSecond) }, + { "-inserts", CMDLINE_INT, offsetof(Config, nIPT) }, + { "-threads", CMDLINE_INT, offsetof(Config, nThread) }, + { "-mutex", CMDLINE_BOOL, offsetof(Config, bMutex) }, + { "-rm", CMDLINE_BOOL, offsetof(Config, bRm) }, + { "-autockpt",CMDLINE_INT, offsetof(Config, nAutoCkpt) }, + { "-mmap", CMDLINE_INT, offsetof(Config, nMmap) }, + { "-clear-cache", CMDLINE_BOOL, offsetof(Config, bClearCache) }, + { "-file", CMDLINE_STRING, offsetof(Config, zFile) }, + { "-osinst", CMDLINE_BOOL, offsetof(Config, bOsinst) }, { 0, 0, 0 } }; + conf.nAutoCkpt = 1000; cmdline_process(apArg, argc, argv, (void*)&conf); if( err.rc==SQLITE_OK ){ char *z = cmdline_construct(apArg, (void*)&conf); diff --git a/test/tt3_core.c b/test/tt3_core.c index dffadac9c8..31612e9cf6 100644 --- a/test/tt3_core.c +++ b/test/tt3_core.c @@ -166,6 +166,11 @@ static void cmdline_process( int n = strlen(z); int iOpt = -1; + if( z[0]=='-' && z[1]=='-' ){ + z++; + n--; + } + for(iArg=0; apArg[iArg].zSwitch; iArg++){ if( 0==sqlite3_strnicmp(apArg[iArg].zSwitch, z, n) ){ if( iOpt>=0 ){ From 21ac1de75b5da94c858d599240648d66fdcee5c6 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 6 Jul 2016 08:32:53 +0000 Subject: [PATCH 036/179] Fix a typo in test program bc_test1.c. FossilOrigin-Name: 2c61b7ab1808d5576a21fea84c3f414db51abfa5 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/bc_test1.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 9eeb419b53..20610275e2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\stweaks\sto\sthe\sbc_test1\stest\sprogram. -D 2016-05-30T05:45:32.809 +C Fix\sa\stypo\sin\stest\sprogram\sbc_test1.c. +D 2016-07-06T08:32:53.109 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -521,7 +521,7 @@ F test/backup_malloc.test 7162d604ec2b4683c4b3799a48657fb8b5e2d450 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f F test/badutf2.test f5bc7f2d280670ecd79b9cf4f0f1760c607fe51f F test/bc_common.tcl b5e42d80305be95697e6370e015af571e5333a1c -F test/bc_test1.c 64b3438cbb0a267de11c2c01e9ec7a96ebdbc3a3 +F test/bc_test1.c e0a092579552e066ed4ce7bcdaecfa69c4aacc8d F test/bestindex1.test 0cf1bd2d7b97d3a3a8c10736125274f64765c4ee F test/bestindex2.test 4a06b8922ab2fd09434870da8d1cdf525aaf7060 F test/between.test 34d375fb5ce1ae283ffe82b6b233e9f38e84fc6c @@ -1491,7 +1491,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5528de4a53c19557798b6169e1d738f1a301e131 -R 4e1a86c5f70b95ae38ae7d31e017c12f +P d0d0bab4e92402b6af98366be6e8955588613a51 +R fd9335f42062b54523dedc8b82de076d U dan -Z f49d8d592868742b4ebcfcd7d10075da +Z 5f55bfec2b6a0fc682708f8a8d6277b4 diff --git a/manifest.uuid b/manifest.uuid index 250d42f7a8..aa42780861 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d0d0bab4e92402b6af98366be6e8955588613a51 \ No newline at end of file +2c61b7ab1808d5576a21fea84c3f414db51abfa5 \ No newline at end of file diff --git a/test/bc_test1.c b/test/bc_test1.c index 6fbb7ca8e9..89135d66d0 100644 --- a/test/bc_test1.c +++ b/test/bc_test1.c @@ -410,7 +410,7 @@ static char *thread_main(int iTid, void *pArg){ sql_script_printf(&err, &db, "PRAGMA wal_autocheckpoint = 0;" "PRAGMA synchronous = 0;" - "PRAGMA mmap_limit = %lld;", + "PRAGMA mmap_size = %lld;", (i64)(pConfig->nMmap) * 1024 * 1024 ); From c18d304066437f1ba3f9af9caa0fd544b0d229a4 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 22 Aug 2016 20:49:06 +0000 Subject: [PATCH 037/179] Add an experimental module to detect conflicts between sessions changesets. FossilOrigin-Name: 0c9fd6b723041955b5182caa430312e5124fdc83 --- ext/session/changebatch1.test | 55 ++++ ext/session/sqlite3changebatch.c | 442 +++++++++++++++++++++++++++++++ ext/session/sqlite3changebatch.h | 75 ++++++ ext/session/test_session.c | 127 ++++++++- main.mk | 1 + manifest | 23 +- manifest.uuid | 2 +- 7 files changed, 714 insertions(+), 11 deletions(-) create mode 100644 ext/session/changebatch1.test create mode 100644 ext/session/sqlite3changebatch.c create mode 100644 ext/session/sqlite3changebatch.h diff --git a/ext/session/changebatch1.test b/ext/session/changebatch1.test new file mode 100644 index 0000000000..6e9d1a9074 --- /dev/null +++ b/ext/session/changebatch1.test @@ -0,0 +1,55 @@ +# 2016 August 23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix changebatch1 + +proc do_changebatch_test {tn args} { + set C [list] + foreach a $args { + sqlite3session S db main + S attach * + execsql $a + lappend C [S changeset] + S delete + } + + sqlite3changebatch cb db + set i 1 + foreach ::cs [lrange $C 0 end-1] { + do_test $tn.$i { cb add [set ::cs] } SQLITE_OK + incr i + } + + set ::cs [lindex $C end] + do_test $tn.$i { cb add [set ::cs] } SQLITE_CONSTRAINT + + cb delete +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a PRIMARY KEY, b); +} + +do_changebatch_test 1.1 { + INSERT INTO t1 VALUES(1, 1); +} { + DELETE FROM t1 WHERE a=1; +} + +finish_test diff --git a/ext/session/sqlite3changebatch.c b/ext/session/sqlite3changebatch.c new file mode 100644 index 0000000000..c0f9b28340 --- /dev/null +++ b/ext/session/sqlite3changebatch.c @@ -0,0 +1,442 @@ + +#if !defined(SQLITE_TEST) || (defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK)) + +#include "sqlite3session.h" +#include "sqlite3changebatch.h" + +#include +#include + +typedef struct BatchTable BatchTable; +typedef struct BatchIndex BatchIndex; +typedef struct BatchIndexEntry BatchIndexEntry; +typedef struct BatchHash BatchHash; + +struct sqlite3_changebatch { + sqlite3 *db; /* Database handle used to read schema */ + BatchTable *pTab; /* First in linked list of tables */ + int iChangesetId; /* Current changeset id */ + int iNextIdxId; /* Next available index id */ + int nEntry; /* Number of entries in hash table */ + int nHash; /* Number of hash buckets */ + BatchIndexEntry **apHash; /* Array of hash buckets */ +}; + +struct BatchTable { + BatchIndex *pIdx; /* First in linked list of UNIQUE indexes */ + BatchTable *pNext; /* Next table */ + char zTab[1]; /* Table name */ +}; + +struct BatchIndex { + BatchIndex *pNext; /* Next index on same table */ + int iId; /* Index id (assigned internally) */ + int bPk; /* True for PK index */ + int nCol; /* Size of aiCol[] array */ + int *aiCol; /* Array of columns that make up index */ +}; + +struct BatchIndexEntry { + BatchIndexEntry *pNext; /* Next colliding hash table entry */ + int iChangesetId; /* Id of associated changeset */ + int iIdxId; /* Id of index this key is from */ + int szRecord; + char aRecord[1]; +}; + +/* +** Allocate and zero a block of nByte bytes. Must be freed using cbFree(). +*/ +static void *cbMalloc(int *pRc, int nByte){ + void *pRet; + + if( *pRc ){ + pRet = 0; + }else{ + pRet = sqlite3_malloc(nByte); + if( pRet ){ + memset(pRet, 0, nByte); + }else{ + *pRc = SQLITE_NOMEM; + } + } + + return pRet; +} + +/* +** Free an allocation made by cbMalloc(). +*/ +static void cbFree(void *p){ + sqlite3_free(p); +} + +/* +** Return the hash bucket that pEntry belongs in. +*/ +static int cbHash(sqlite3_changebatch *p, BatchIndexEntry *pEntry){ + unsigned int iHash = (unsigned int)pEntry->iIdxId; + unsigned char *pEnd = (unsigned char*)&pEntry->aRecord[pEntry->szRecord]; + unsigned char *pIter; + + for(pIter=pEntry->aRecord; pIternHash); +} + +/* +** Resize the hash table. +*/ +static int cbHashResize(sqlite3_changebatch *p){ + int rc = SQLITE_OK; + BatchIndexEntry **apNew; + int nNew = (p->nHash ? p->nHash*2 : 512); + int i; + + apNew = cbMalloc(&rc, sizeof(BatchIndexEntry*) * nNew); + if( rc==SQLITE_OK ){ + int nHash = p->nHash; + p->nHash = nNew; + for(i=0; iapHash[i] ){ + int iHash = cbHash(p, pEntry); + p->apHash[i] = pEntry->pNext; + pEntry->pNext = apNew[iHash]; + apNew[iHash] = pEntry; + } + } + + cbFree(p->apHash); + p->apHash = apNew; + } + + return rc; +} + + +/* +** Allocate a new sqlite3_changebatch object. +*/ +int sqlite3changebatch_new(sqlite3 *db, sqlite3_changebatch **pp){ + sqlite3_changebatch *pRet; + int rc = SQLITE_OK; + *pp = pRet = (sqlite3_changebatch*)cbMalloc(&rc, sizeof(sqlite3_changebatch)); + if( pRet ){ + pRet->db = db; + } + return rc; +} + +/* +** Add a BatchIndex entry for index zIdx to table pTab. +*/ +static int cbAddIndex( + sqlite3_changebatch *p, + BatchTable *pTab, + const char *zIdx, + int bPk +){ + int nCol = 0; + sqlite3_stmt *pIndexInfo = 0; + BatchIndex *pNew = 0; + int rc; + char *zIndexInfo; + + zIndexInfo = (char*)sqlite3_mprintf("PRAGMA main.index_info = %Q", zIdx); + if( zIndexInfo ){ + rc = sqlite3_prepare_v2(p->db, zIndexInfo, -1, &pIndexInfo, 0); + sqlite3_free(zIndexInfo); + }else{ + rc = SQLITE_NOMEM; + } + + if( rc==SQLITE_OK ){ + int rc2; + while( SQLITE_ROW==sqlite3_step(pIndexInfo) ){ nCol++; } + rc2 = sqlite3_reset(pIndexInfo); + if( rc==SQLITE_OK ) rc = rc2; + } + + pNew = (BatchIndex*)cbMalloc(&rc, sizeof(BatchIndex) + sizeof(int) * nCol); + if( rc==SQLITE_OK ){ + int rc2; + pNew->nCol = nCol; + pNew->bPk = bPk; + pNew->aiCol = (int*)&pNew[1]; + pNew->iId = p->iNextIdxId++; + while( SQLITE_ROW==sqlite3_step(pIndexInfo) ){ + int i = sqlite3_column_int(pIndexInfo, 0); + int j = sqlite3_column_int(pIndexInfo, 1); + pNew->aiCol[i] = j; + } + rc2 = sqlite3_reset(pIndexInfo); + if( rc==SQLITE_OK ) rc = rc2; + } + + if( rc==SQLITE_OK ){ + pNew->pNext = pTab->pIdx; + pTab->pIdx = pNew; + }else{ + cbFree(pNew); + } + sqlite3_finalize(pIndexInfo); + + return rc; +} + +/* +** Find or create the BatchTable object named zTab. +*/ +static int cbFindTable( + sqlite3_changebatch *p, + const char *zTab, + BatchTable **ppTab +){ + BatchTable *pRet = 0; + int rc = SQLITE_OK; + + for(pRet=p->pTab; pRet; pRet=pRet->pNext){ + if( 0==sqlite3_stricmp(zTab, pRet->zTab) ) break; + } + + if( pRet==0 ){ + int nTab = strlen(zTab); + pRet = (BatchTable*)cbMalloc(&rc, nTab + sizeof(BatchTable)); + if( pRet ){ + sqlite3_stmt *pIndexList = 0; + char *zIndexList = 0; + int rc2; + memcpy(pRet->zTab, zTab, nTab); + + zIndexList = sqlite3_mprintf("PRAGMA main.index_list = %Q", zTab); + if( zIndexList==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zIndexList, -1, &pIndexList, 0); + sqlite3_free(zIndexList); + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pIndexList) ){ + if( sqlite3_column_int(pIndexList, 2) ){ + const char *zIdx = (const char*)sqlite3_column_text(pIndexList, 1); + const char *zTyp = (const char*)sqlite3_column_text(pIndexList, 3); + rc = cbAddIndex(p, pRet, zIdx, (zTyp[0]=='p')); + } + } + rc2 = sqlite3_finalize(pIndexList); + if( rc==SQLITE_OK ) rc = rc2; + + if( rc==SQLITE_OK ){ + pRet->pNext = p->pTab; + p->pTab = pRet; + } + } + } + + *ppTab = pRet; + return rc; +} + +static int cbAddToHash( + sqlite3_changebatch *p, + sqlite3_changeset_iter *pIter, + BatchIndex *pIdx, + int (*xVal)(sqlite3_changeset_iter*,int,sqlite3_value**), + int *pbConf +){ + BatchIndexEntry *pNew; + int sz = pIdx->nCol; + int i; + int iOut = 0; + int rc = SQLITE_OK; + + for(i=0; rc==SQLITE_OK && inCol; i++){ + sqlite3_value *pVal; + rc = xVal(pIter, pIdx->aiCol[i], &pVal); + if( rc==SQLITE_OK ){ + int eType = 0; + if( pVal ){ + eType = sqlite3_value_type(pVal); + } + switch( eType ){ + case 0: + case SQLITE_NULL: + return SQLITE_OK; + + case SQLITE_INTEGER: + sz += 8; + break; + case SQLITE_FLOAT: + sz += 8; + break; + + default: + assert( eType==SQLITE_TEXT || eType==SQLITE_BLOB ); + sz += sqlite3_value_bytes(pVal); + break; + } + } + } + + pNew = cbMalloc(&rc, sizeof(BatchIndexEntry) + sz); + if( pNew ){ + pNew->iChangesetId = p->iChangesetId; + pNew->iIdxId = pIdx->iId; + pNew->szRecord = sz; + + for(i=0; rc==SQLITE_OK && inCol; i++){ + sqlite3_value *pVal; + rc = xVal(pIter, pIdx->aiCol[i], &pVal); + if( rc==SQLITE_OK ){ + int eType = sqlite3_value_type(pVal); + pNew->aRecord[iOut++] = eType; + switch( eType ){ + case SQLITE_INTEGER: { + sqlite3_int64 i64 = sqlite3_value_int64(pVal); + memcpy(&pNew->aRecord[iOut], &i64, 8); + iOut += 8; + break; + } + case SQLITE_FLOAT: { + double d64 = sqlite3_value_double(pVal); + memcpy(&pNew->aRecord[iOut], &d64, sizeof(double)); + iOut += sizeof(double); + break; + } + + default: { + int nByte = sqlite3_value_bytes(pVal); + const char *z = (const char*)sqlite3_value_blob(pVal); + memcpy(&pNew->aRecord[iOut], z, nByte); + iOut += nByte; + break; + } + } + } + } + } + + if( rc==SQLITE_OK && p->nEntry>=(p->nHash/2) ){ + rc = cbHashResize(p); + } + + if( rc==SQLITE_OK ){ + BatchIndexEntry *pIter; + int iHash = cbHash(p, pNew); + + assert( iHash>=0 && iHashnHash ); + for(pIter=p->apHash[iHash]; pIter; pIter=pIter->pNext){ + if( pNew->szRecord==pIter->szRecord + && 0==memcmp(pNew->aRecord, pIter->aRecord, pNew->szRecord) + ){ + if( pNew->iChangesetId!=pIter->iChangesetId ){ + *pbConf = 1; + } + cbFree(pNew); + pNew = 0; + break; + } + } + + if( pNew ){ + pNew->pNext = p->apHash[iHash]; + p->apHash[iHash] = pNew; + p->nEntry++; + } + } + + p->iChangesetId++; + return rc; +} + + +/* +** Add a changeset to the current batch. +*/ +int sqlite3changebatch_add(sqlite3_changebatch *p, void *pBuf, int nBuf){ + sqlite3_changeset_iter *pIter; /* Iterator opened on pBuf/nBuf */ + int rc; /* Return code */ + int bConf = 0; /* Conflict was detected */ + + rc = sqlite3changeset_start(&pIter, nBuf, pBuf); + if( rc==SQLITE_OK ){ + int rc2; + for(rc2 = sqlite3changeset_next(pIter); + rc2==SQLITE_ROW; + rc2 = sqlite3changeset_next(pIter) + ){ + BatchTable *pTab; + BatchIndex *pIdx; + const char *zTab; /* Table this change applies to */ + int nCol; /* Number of columns in table */ + int op; /* UPDATE, INSERT or DELETE */ + + sqlite3changeset_op(pIter, &zTab, &nCol, &op, 0); + assert( op==SQLITE_INSERT || op==SQLITE_UPDATE || op==SQLITE_DELETE ); + + rc = cbFindTable(p, zTab, &pTab); + for(pIdx=pTab->pIdx; pIdx && rc==SQLITE_OK; pIdx=pIdx->pNext){ + if( op==SQLITE_UPDATE && pIdx->bPk ) continue; + if( op==SQLITE_UPDATE || op==SQLITE_DELETE ){ + rc = cbAddToHash(p, pIter, pIdx, sqlite3changeset_old, &bConf); + } + if( op==SQLITE_UPDATE || op==SQLITE_INSERT ){ + rc = cbAddToHash(p, pIter, pIdx, sqlite3changeset_new, &bConf); + } + } + if( rc!=SQLITE_OK ) break; + } + + rc2 = sqlite3changeset_finalize(pIter); + if( rc==SQLITE_OK ) rc = rc2; + } + + if( rc==SQLITE_OK && bConf ){ + rc = SQLITE_CONSTRAINT; + } + return rc; +} + +/* +** Zero an existing changebatch object. +*/ +void sqlite3changebatch_zero(sqlite3_changebatch *p){ + int i; + for(i=0; inHash; i++){ + BatchIndexEntry *pEntry; + BatchIndexEntry *pNext; + for(pEntry=p->apHash[i]; pEntry; pEntry=pNext){ + pNext = pEntry->pNext; + cbFree(pEntry); + } + } + cbFree(p->apHash); + p->nHash = 0; + p->apHash = 0; +} + +/* +** Delete a changebatch object. +*/ +void sqlite3changebatch_delete(sqlite3_changebatch *p){ + BatchTable *pTab; + BatchTable *pTabNext; + + sqlite3changebatch_zero(p); + for(pTab=p->pTab; pTab; pTab=pTabNext){ + BatchIndex *pIdx; + BatchIndex *pIdxNext; + for(pIdx=pTab->pIdx; pIdx; pIdx=pIdxNext){ + pIdxNext = pIdx->pNext; + cbFree(pIdx); + } + pTabNext = pTab->pNext; + cbFree(pTab); + } + cbFree(p); +} + +#endif /* SQLITE_ENABLE_SESSION && SQLITE_ENABLE_PREUPDATE_HOOK */ diff --git a/ext/session/sqlite3changebatch.h b/ext/session/sqlite3changebatch.h new file mode 100644 index 0000000000..9afa6c174e --- /dev/null +++ b/ext/session/sqlite3changebatch.h @@ -0,0 +1,75 @@ + +#if !defined(SQLITECHANGEBATCH_H_) +#define SQLITECHANGEBATCH_H_ 1 + +typedef struct sqlite3_changebatch sqlite3_changebatch; + +/* +** Create a new changebatch object for detecting conflicts between +** changesets associated with a schema equivalent to that of the "main" +** database of the open database handle db passed as the first +** parameter. It is the responsibility of the caller to ensure that +** the database handle is not closed until after the changebatch +** object has been deleted. +** +** A changebatch object is used to detect batches of non-conflicting +** changesets. Changesets that do not conflict may be applied to the +** target database in any order without affecting the final state of +** the database. +** +** The changebatch object only works reliably if PRIMARY KEY and UNIQUE +** constraints on tables affected by the changesets use collation +** sequences that are equivalent to built-in collation sequence +** BINARY for the == operation. +** +** If successful, SQLITE_OK is returned and (*pp) set to point to +** the new changebatch object. If an error occurs, an SQLite error +** code is returned and the final value of (*pp) is undefined. +*/ +int sqlite3changebatch_new(sqlite3 *db, sqlite3_changebatch **pp); + +/* +** Argument p points to a buffer containing a changeset n bytes in +** size. Assuming no error occurs, this function returns SQLITE_OK +** if the changeset does not conflict with any changeset passed +** to an sqlite3changebatch_add() call made on the same +** sqlite3_changebatch* handle since the most recent call to +** sqlite3changebatch_zero(). If the changeset does conflict with +** an earlier such changeset, SQLITE_CONSTRAINT is returned. Or, +** if an error occurs, some other SQLite error code may be returned. +** +** One changeset is said to conflict with another if +** either: +** +** * the two changesets contain operations (INSERT, UPDATE or +** DELETE) on the same row, identified by primary key, or +** +** * the two changesets contain operations (INSERT, UPDATE or +** DELETE) on rows with identical values in any combination +** of fields constrained by a UNIQUE constraint. +** +** Even if this function returns SQLITE_CONFLICT, the current +** changeset is added to the internal data structures - so future +** calls to this function may conflict with it. If this function +** returns any result code other than SQLITE_OK or SQLITE_CONFLICT, +** the result of any future call to sqlite3changebatch_add() is +** undefined. +** +** Only changesets may be passed to this function. Passing a +** patchset to this function results in an SQLITE_MISUSE error. +*/ +int sqlite3changebatch_add(sqlite3_changebatch*, void *p, int n); + +/* +** Zero a changebatch object. This causes the records of all earlier +** calls to sqlite3changebatch_add() to be discarded. +*/ +void sqlite3changebatch_zero(sqlite3_changebatch*); + +/* +** Delete a changebatch object. +*/ +void sqlite3changebatch_delete(sqlite3_changebatch*); + +#endif /* !defined(SQLITECHANGEBATCH_H_) */ + diff --git a/ext/session/test_session.c b/ext/session/test_session.c index 103a1c2d7a..3f4cdd3c34 100644 --- a/ext/session/test_session.c +++ b/ext/session/test_session.c @@ -373,7 +373,7 @@ static int test_filter_handler( Tcl_DecrRefCount(pEval); return res; -} +} static int test_conflict_handler( void *pCtx, /* Pointer to TestConflictHandler structure */ @@ -918,6 +918,127 @@ static int SQLITE_TCLAPI test_sqlite3session_foreach( return TCL_OK; } +#include "sqlite3changebatch.h" + +typedef struct TestChangebatch TestChangebatch; +struct TestChangebatch { + sqlite3_changebatch *pChangebatch; +}; + +/* +** Tclcmd: $changebatch add BLOB +** $changebatch zero +** $changebatch delete +*/ +static int SQLITE_TCLAPI test_changebatch_cmd( + void *clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + TestChangebatch *p = (TestChangebatch*)clientData; + sqlite3_changebatch *pChangebatch = p->pChangebatch; + struct SessionSubcmd { + const char *zSub; + int nArg; + const char *zMsg; + int iSub; + } aSub[] = { + { "add", 1, "CHANGESET", }, /* 0 */ + { "zero", 0, "", }, /* 1 */ + { "delete", 0, "", }, /* 2 */ + { 0 } + }; + int iSub; + int rc; + + if( objc<2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "SUBCOMMAND ..."); + return TCL_ERROR; + } + rc = Tcl_GetIndexFromObjStruct(interp, + objv[1], aSub, sizeof(aSub[0]), "sub-command", 0, &iSub + ); + if( rc!=TCL_OK ) return rc; + if( objc!=2+aSub[iSub].nArg ){ + Tcl_WrongNumArgs(interp, 2, objv, aSub[iSub].zMsg); + return TCL_ERROR; + } + + switch( iSub ){ + case 0: { /* add */ + int nArg; + unsigned char *pArg = Tcl_GetByteArrayFromObj(objv[2], &nArg); + rc = sqlite3changebatch_add(pChangebatch, pArg, nArg); + if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT ){ + return test_session_error(interp, rc, 0); + }else{ + extern const char *sqlite3ErrName(int); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + } + break; + } + + case 1: { /* zero */ + sqlite3changebatch_zero(pChangebatch); + break; + } + + case 2: /* delete */ + Tcl_DeleteCommand(interp, Tcl_GetString(objv[0])); + break; + } + + return TCL_OK; +} + +static void SQLITE_TCLAPI test_changebatch_del(void *clientData){ + TestChangebatch *p = (TestChangebatch*)clientData; + sqlite3changebatch_delete(p->pChangebatch); + ckfree((char*)p); +} + +/* +** Tclcmd: sqlite3changebatch CMD DB-HANDLE +*/ +static int SQLITE_TCLAPI test_sqlite3changebatch( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db; + Tcl_CmdInfo info; + int rc; /* sqlite3session_create() return code */ + TestChangebatch *p; /* New wrapper object */ + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "CMD DB-HANDLE"); + return TCL_ERROR; + } + + if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(objv[2]), &info) ){ + Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(objv[2]), 0); + return TCL_ERROR; + } + db = *(sqlite3 **)info.objClientData; + + p = (TestChangebatch*)ckalloc(sizeof(TestChangebatch)); + memset(p, 0, sizeof(TestChangebatch)); + rc = sqlite3changebatch_new(db, &p->pChangebatch); + if( rc!=SQLITE_OK ){ + ckfree((char*)p); + return test_session_error(interp, rc, 0); + } + + Tcl_CreateObjCommand( + interp, Tcl_GetString(objv[1]), test_changebatch_cmd, (ClientData)p, + test_changebatch_del + ); + Tcl_SetObjResult(interp, objv[1]); + return TCL_OK; +} + int TestSession_Init(Tcl_Interp *interp){ Tcl_CreateObjCommand(interp, "sqlite3session", test_sqlite3session, 0, 0); Tcl_CreateObjCommand( @@ -936,6 +1057,10 @@ int TestSession_Init(Tcl_Interp *interp){ interp, "sqlite3changeset_apply_replace_all", test_sqlite3changeset_apply_replace_all, 0, 0 ); + + Tcl_CreateObjCommand( + interp, "sqlite3changebatch", test_sqlite3changebatch, 0, 0 + ); return TCL_OK; } diff --git a/main.mk b/main.mk index 451837ffff..034b65332a 100644 --- a/main.mk +++ b/main.mk @@ -390,6 +390,7 @@ TESTSRC2 = \ $(TOP)/ext/fts3/fts3_write.c \ $(TOP)/ext/async/sqlite3async.c \ $(TOP)/ext/session/sqlite3session.c \ + $(TOP)/ext/session/sqlite3changebatch.c \ $(TOP)/ext/session/test_session.c # Header files used by all library source files. diff --git a/manifest b/manifest index 2c686caa9b..698acd2427 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sVACUUM\scommand\sso\sthat\sit\scan\soperate\son\san\sattached\sdatabase. -D 2016-08-19T15:15:55.612 +C Add\san\sexperimental\smodule\sto\sdetect\sconflicts\sbetween\ssessions\schangesets. +D 2016-08-22T20:49:06.286 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc d66d0395c38571aab3804f8db0fa20707ae4609a @@ -281,6 +281,7 @@ F ext/rtree/rtree_util.tcl 06aab2ed5b826545bf215fff90ecb9255a8647ea F ext/rtree/sqlite3rtree.h 9c5777af3d2921c7b4ae4954e8e5697502289d28 F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 +F ext/session/changebatch1.test 5a69388b91ad02544ef2d9ae27771dbf0f9cea58 F ext/session/changeset.c 4ccbaa4531944c24584bf6a61ba3a39c62b6267a F ext/session/session1.test 98f384736e2bc21ccf5ed81bdadcff4ad863393b F ext/session/session2.test 284de45abae4cc1082bc52012ee81521d5ac58e0 @@ -300,16 +301,18 @@ F ext/session/sessionG.test 01ef705096a9d3984eebdcca79807a211dee1b60 F ext/session/session_common.tcl a1293167d14774b5e728836720497f40fe4ea596 F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7 F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0 +F ext/session/sqlite3changebatch.c 37fb1c87d7b3ccaff194411ff8344d9cc056bd98 +F ext/session/sqlite3changebatch.h 50a302e4fc535324309607b13a1993bca074758b F ext/session/sqlite3session.c 37485891b4add26cf61495df193c419f36556a32 F ext/session/sqlite3session.h 69bf73cfd71e58f2ae5d2aa935b2c1a541aee555 -F ext/session/test_session.c 2caed9a659586428c63ca46e4900347b374487d4 +F ext/session/test_session.c 24968972a5709d2ccf5d570f1a13ce009fbc0d86 F ext/userauth/sqlite3userauth.h 19cb6f0e31316d0ee4afdfb7a85ef9da3333a220 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 1883ecab643b136e8ab3fdc33785e6ea8b5ceb46 +F main.mk de9447348ea580282aa47dbffd20b042bfbec4e1 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -1511,8 +1514,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cb9865e14db1c0076618f13400151112f84960cb ad35ef116296e5d6aaeb9ef260bf35bee3bd6d20 -R abd29f9f3d87c2af53d085425233b2a6 -T +closed ad35ef116296e5d6aaeb9ef260bf35bee3bd6d20 -U drh -Z a6de375e7bb722769082d6e640b9c49b +P 083f9e6270fa4faa402b91231271da4f3915c79f +R c89955dc116a299d44f6360524b79a8a +T *branch * changebatch +T *sym-changebatch * +T -sym-trunk * +U dan +Z f9a59b56ef4c3ca7ba5584bf19e5e811 diff --git a/manifest.uuid b/manifest.uuid index 4e9d48c81e..7935283ab4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -083f9e6270fa4faa402b91231271da4f3915c79f \ No newline at end of file +0c9fd6b723041955b5182caa430312e5124fdc83 \ No newline at end of file From a7d16a5cd1fa45e1a57d83d7fd46efa7a0f00621 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 22 Aug 2016 21:01:39 +0000 Subject: [PATCH 038/179] Add a couple of extra tests to changebatch1.test. FossilOrigin-Name: 207d970b7956c38af42c389b91a741a68b2c4eec --- ext/session/changebatch1.test | 35 +++++++++++++++++++++++++++++++++++ manifest | 15 ++++++--------- manifest.uuid | 2 +- 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/ext/session/changebatch1.test b/ext/session/changebatch1.test index 6e9d1a9074..056512aca3 100644 --- a/ext/session/changebatch1.test +++ b/ext/session/changebatch1.test @@ -52,4 +52,39 @@ do_changebatch_test 1.1 { DELETE FROM t1 WHERE a=1; } +do_execsql_test 1.2.0 { + INSERT INTO t1 VALUES(1, 1); + INSERT INTO t1 VALUES(2, 2); + INSERT INTO t1 VALUES(3, 3); +} +do_changebatch_test 1.2.1 { + DELETE FROM t1 WHERE a=2; +} { + INSERT INTO t1 VALUES(2, 2); +} + +#------------------------------------------------------------------------- +do_execsql_test 2.0 { + CREATE TABLE x1(a, b PRIMARY KEY, c UNIQUE); + CREATE TABLE x2(a PRIMARY KEY, b UNIQUE, c UNIQUE); + + INSERT INTO x1 VALUES(1, 1, 'a'); + INSERT INTO x1 VALUES(1, 2, 'b'); + INSERT INTO x1 VALUES(1, 3, 'c'); +} + +do_changebatch_test 2.1 { + DELETE FROM x1 WHERE b=2; +} { + UPDATE x1 SET c='b' WHERE b=3; +} + +do_changebatch_test 2.2 { + DELETE FROM x1 WHERE b=1; +} { + INSERT INTO x1 VALUES(1, 5, 'a'); +} + finish_test + + diff --git a/manifest b/manifest index 698acd2427..d16bdf85a5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sexperimental\smodule\sto\sdetect\sconflicts\sbetween\ssessions\schangesets. -D 2016-08-22T20:49:06.286 +C Add\sa\scouple\sof\sextra\stests\sto\schangebatch1.test. +D 2016-08-22T21:01:39.413 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc d66d0395c38571aab3804f8db0fa20707ae4609a @@ -281,7 +281,7 @@ F ext/rtree/rtree_util.tcl 06aab2ed5b826545bf215fff90ecb9255a8647ea F ext/rtree/sqlite3rtree.h 9c5777af3d2921c7b4ae4954e8e5697502289d28 F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 -F ext/session/changebatch1.test 5a69388b91ad02544ef2d9ae27771dbf0f9cea58 +F ext/session/changebatch1.test b2d1a8c8b8f86881f50b481eae233f8abfb61436 F ext/session/changeset.c 4ccbaa4531944c24584bf6a61ba3a39c62b6267a F ext/session/session1.test 98f384736e2bc21ccf5ed81bdadcff4ad863393b F ext/session/session2.test 284de45abae4cc1082bc52012ee81521d5ac58e0 @@ -1514,10 +1514,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 083f9e6270fa4faa402b91231271da4f3915c79f -R c89955dc116a299d44f6360524b79a8a -T *branch * changebatch -T *sym-changebatch * -T -sym-trunk * +P 0c9fd6b723041955b5182caa430312e5124fdc83 +R 0bf0e85497e377afe1f85fb7abd8225d U dan -Z f9a59b56ef4c3ca7ba5584bf19e5e811 +Z cf85cb19b321088cfc5e96ee875ee34c diff --git a/manifest.uuid b/manifest.uuid index 7935283ab4..f8bc749f9d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0c9fd6b723041955b5182caa430312e5124fdc83 \ No newline at end of file +207d970b7956c38af42c389b91a741a68b2c4eec \ No newline at end of file From 8bbf544747d4b458f301b8aa08d20c29fb5a61d1 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 23 Aug 2016 17:02:28 +0000 Subject: [PATCH 039/179] Add a new method to sessions - sqlite3sessions_fullchangeset() - to return a changeset that always contains values for all old.* fields. Update changebatch to use these values to more reliably detect multi-column UNIQUE constraint violations. FossilOrigin-Name: efa761b2f509844b9212dd20bf0d082c6338e83f --- ext/session/changebatch1.test | 145 ++++++++++++++++++++++--------- ext/session/sqlite3changebatch.c | 29 +++++-- ext/session/sqlite3session.c | 56 ++++++++---- ext/session/sqlite3session.h | 13 +++ ext/session/test_session.c | 25 +++--- manifest | 20 ++--- manifest.uuid | 2 +- 7 files changed, 206 insertions(+), 84 deletions(-) diff --git a/ext/session/changebatch1.test b/ext/session/changebatch1.test index 056512aca3..fc3f50ab4c 100644 --- a/ext/session/changebatch1.test +++ b/ext/session/changebatch1.test @@ -19,14 +19,19 @@ ifcapable !session {finish_test; return} set testprefix changebatch1 -proc do_changebatch_test {tn args} { +proc sql_to_changeset {method sql} { + sqlite3session S db main + S attach * + execsql $sql + set ret [S $method] + S delete + return $ret +} + +proc do_changebatch_test {tn method args} { set C [list] foreach a $args { - sqlite3session S db main - S attach * - execsql $a - lappend C [S changeset] - S delete + lappend C [sql_to_changeset $method $a] } sqlite3changebatch cb db @@ -42,49 +47,111 @@ proc do_changebatch_test {tn args} { cb delete } -do_execsql_test 1.0 { - CREATE TABLE t1(a PRIMARY KEY, b); +proc do_changebatch_test1 {tn args} { + uplevel do_changebatch_test $tn changeset $args } - -do_changebatch_test 1.1 { - INSERT INTO t1 VALUES(1, 1); -} { - DELETE FROM t1 WHERE a=1; +proc do_changebatch_test2 {tn args} { + uplevel do_changebatch_test $tn fullchangeset $args } -do_execsql_test 1.2.0 { - INSERT INTO t1 VALUES(1, 1); - INSERT INTO t1 VALUES(2, 2); - INSERT INTO t1 VALUES(3, 3); -} -do_changebatch_test 1.2.1 { - DELETE FROM t1 WHERE a=2; +#------------------------------------------------------------------------- +# The body of the following loop contains tests for database schemas +# that do not feature multi-column UNIQUE constraints. In this case +# it doesn't matter if the changesets are generated using +# sqlite3session_changeset() or sqlite3session_fullchangeset(). +# +foreach {tn testfunction} { + 1 do_changebatch_test1 + 2 do_changebatch_test2 } { - INSERT INTO t1 VALUES(2, 2); + reset_db + + #------------------------------------------------------------------------- + # + do_execsql_test $tn.1.0 { + CREATE TABLE t1(a PRIMARY KEY, b); + } + + $testfunction $tn.1.1 { + INSERT INTO t1 VALUES(1, 1); + } { + DELETE FROM t1 WHERE a=1; + } + + do_execsql_test $tn.1.2.0 { + INSERT INTO t1 VALUES(1, 1); + INSERT INTO t1 VALUES(2, 2); + INSERT INTO t1 VALUES(3, 3); + } + $testfunction $tn.1.2.1 { + DELETE FROM t1 WHERE a=2; + } { + INSERT INTO t1 VALUES(2, 2); + } + + #------------------------------------------------------------------------- + # + do_execsql_test $tn.2.0 { + CREATE TABLE x1(a, b PRIMARY KEY, c UNIQUE); + CREATE TABLE x2(a PRIMARY KEY, b UNIQUE, c UNIQUE); + + INSERT INTO x1 VALUES(1, 1, 'a'); + INSERT INTO x1 VALUES(1, 2, 'b'); + INSERT INTO x1 VALUES(1, 3, 'c'); + } + + $testfunction $tn.2.1 { + DELETE FROM x1 WHERE b=2; + } { + UPDATE x1 SET c='b' WHERE b=3; + } + + $testfunction $tn.2.2 { + DELETE FROM x1 WHERE b=1; + } { + INSERT INTO x1 VALUES(1, 5, 'a'); + } } #------------------------------------------------------------------------- -do_execsql_test 2.0 { - CREATE TABLE x1(a, b PRIMARY KEY, c UNIQUE); - CREATE TABLE x2(a PRIMARY KEY, b UNIQUE, c UNIQUE); - - INSERT INTO x1 VALUES(1, 1, 'a'); - INSERT INTO x1 VALUES(1, 2, 'b'); - INSERT INTO x1 VALUES(1, 3, 'c'); +# Test some multi-column UNIQUE constraints. First Using _changeset() to +# demonstrate the problem, then using _fullchangeset() to show that it has +# been fixed. +# +reset_db +do_execsql_test 3.0 { + CREATE TABLE y1(a PRIMARY KEY, b, c, UNIQUE(b, c)); + INSERT INTO y1 VALUES(1, 1, 1); + INSERT INTO y1 VALUES(2, 2, 2); + INSERT INTO y1 VALUES(3, 3, 3); + INSERT INTO y1 VALUES(4, 3, 4); + BEGIN; } -do_changebatch_test 2.1 { - DELETE FROM x1 WHERE b=2; -} { - UPDATE x1 SET c='b' WHERE b=3; -} +do_test 3.1.1 { + set c1 [sql_to_changeset changeset { DELETE FROM y1 WHERE a=4 }] + set c2 [sql_to_changeset changeset { UPDATE y1 SET c=4 WHERE a=3 }] + sqlite3changebatch cb db + cb add $c1 + cb add $c2 +} {SQLITE_OK} +do_test 3.1.2 { + cb delete + execsql ROLLBACK +} {} + +do_test 3.1.1 { + set c1 [sql_to_changeset fullchangeset { DELETE FROM y1 WHERE a=4 }] + set c2 [sql_to_changeset fullchangeset { UPDATE y1 SET c=4 WHERE a=3 }] + sqlite3changebatch cb db + cb add $c1 + cb add $c2 +} {SQLITE_CONSTRAINT} +do_test 3.1.2 { + cb delete +} {} -do_changebatch_test 2.2 { - DELETE FROM x1 WHERE b=1; -} { - INSERT INTO x1 VALUES(1, 5, 'a'); -} -finish_test +finish_test diff --git a/ext/session/sqlite3changebatch.c b/ext/session/sqlite3changebatch.c index c0f9b28340..5672a574c5 100644 --- a/ext/session/sqlite3changebatch.c +++ b/ext/session/sqlite3changebatch.c @@ -240,11 +240,26 @@ static int cbFindTable( return rc; } +static int cbGetChangesetValue( + sqlite3_changeset_iter *pIter, + int (*xVal)(sqlite3_changeset_iter*,int,sqlite3_value**), + int (*xFallback)(sqlite3_changeset_iter*,int,sqlite3_value**), + int iVal, + sqlite3_value **ppVal +){ + int rc = xVal(pIter, iVal, ppVal); + if( rc==SQLITE_OK && *ppVal==0 && xFallback ){ + rc = xFallback(pIter, iVal, ppVal); + } + return rc; +} + static int cbAddToHash( sqlite3_changebatch *p, sqlite3_changeset_iter *pIter, BatchIndex *pIdx, int (*xVal)(sqlite3_changeset_iter*,int,sqlite3_value**), + int (*xFallback)(sqlite3_changeset_iter*,int,sqlite3_value**), int *pbConf ){ BatchIndexEntry *pNew; @@ -255,12 +270,10 @@ static int cbAddToHash( for(i=0; rc==SQLITE_OK && inCol; i++){ sqlite3_value *pVal; - rc = xVal(pIter, pIdx->aiCol[i], &pVal); + rc = cbGetChangesetValue(pIter, xVal, xFallback, pIdx->aiCol[i], &pVal); if( rc==SQLITE_OK ){ int eType = 0; - if( pVal ){ - eType = sqlite3_value_type(pVal); - } + if( pVal ) eType = sqlite3_value_type(pVal); switch( eType ){ case 0: case SQLITE_NULL: @@ -289,7 +302,7 @@ static int cbAddToHash( for(i=0; rc==SQLITE_OK && inCol; i++){ sqlite3_value *pVal; - rc = xVal(pIter, pIdx->aiCol[i], &pVal); + rc = cbGetChangesetValue(pIter, xVal, xFallback, pIdx->aiCol[i], &pVal); if( rc==SQLITE_OK ){ int eType = sqlite3_value_type(pVal); pNew->aRecord[iOut++] = eType; @@ -381,10 +394,12 @@ int sqlite3changebatch_add(sqlite3_changebatch *p, void *pBuf, int nBuf){ for(pIdx=pTab->pIdx; pIdx && rc==SQLITE_OK; pIdx=pIdx->pNext){ if( op==SQLITE_UPDATE && pIdx->bPk ) continue; if( op==SQLITE_UPDATE || op==SQLITE_DELETE ){ - rc = cbAddToHash(p, pIter, pIdx, sqlite3changeset_old, &bConf); + rc = cbAddToHash(p, pIter, pIdx, sqlite3changeset_old, 0, &bConf); } if( op==SQLITE_UPDATE || op==SQLITE_INSERT ){ - rc = cbAddToHash(p, pIter, pIdx, sqlite3changeset_new, &bConf); + rc = cbAddToHash(p, pIter, pIdx, + sqlite3changeset_new, sqlite3changeset_old, &bConf + ); } } if( rc!=SQLITE_OK ) break; diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c index 70ca840dae..22728d0b97 100644 --- a/ext/session/sqlite3session.c +++ b/ext/session/sqlite3session.c @@ -25,6 +25,13 @@ typedef struct SessionInput SessionInput; # endif #endif +/* +** The three different types of changesets generated. +*/ +#define SESSIONS_PATCHSET 0 +#define SESSIONS_CHANGESET 1 +#define SESSIONS_FULLCHANGESET 2 + typedef struct SessionHook SessionHook; struct SessionHook { void *pCtx; @@ -1934,7 +1941,7 @@ static void sessionAppendCol( */ static int sessionAppendUpdate( SessionBuffer *pBuf, /* Buffer to append to */ - int bPatchset, /* True for "patchset", 0 for "changeset" */ + int ePatchset, /* True for "patchset", 0 for "changeset" */ sqlite3_stmt *pStmt, /* Statement handle pointing at new row */ SessionChange *p, /* Object containing old values */ u8 *abPK /* Boolean array - true for PK columns */ @@ -1997,8 +2004,8 @@ static int sessionAppendUpdate( /* Add a field to the old.* record. This is omitted if this modules is ** currently generating a patchset. */ - if( bPatchset==0 ){ - if( bChanged || abPK[i] ){ + if( ePatchset!=SESSIONS_PATCHSET ){ + if( ePatchset==SESSIONS_FULLCHANGESET || bChanged || abPK[i] ){ sessionAppendBlob(pBuf, pCsr, nAdvance, &rc); }else{ sessionAppendByte(pBuf, 0, &rc); @@ -2007,7 +2014,7 @@ static int sessionAppendUpdate( /* Add a field to the new.* record. Or the only record if currently ** generating a patchset. */ - if( bChanged || (bPatchset && abPK[i]) ){ + if( bChanged || (ePatchset==SESSIONS_PATCHSET && abPK[i]) ){ sessionAppendCol(&buf2, pStmt, i, &rc); }else{ sessionAppendByte(&buf2, 0, &rc); @@ -2033,7 +2040,7 @@ static int sessionAppendUpdate( */ static int sessionAppendDelete( SessionBuffer *pBuf, /* Buffer to append to */ - int bPatchset, /* True for "patchset", 0 for "changeset" */ + int eChangeset, /* One of SESSIONS_CHANGESET etc. */ SessionChange *p, /* Object containing old values */ int nCol, /* Number of columns in table */ u8 *abPK /* Boolean array - true for PK columns */ @@ -2043,7 +2050,7 @@ static int sessionAppendDelete( sessionAppendByte(pBuf, SQLITE_DELETE, &rc); sessionAppendByte(pBuf, p->bIndirect, &rc); - if( bPatchset==0 ){ + if( eChangeset!=SESSIONS_PATCHSET ){ sessionAppendBlob(pBuf, p->aRecord, p->nRecord, &rc); }else{ int i; @@ -2202,12 +2209,12 @@ static int sessionSelectBind( */ static void sessionAppendTableHdr( SessionBuffer *pBuf, /* Append header to this buffer */ - int bPatchset, /* Use the patchset format if true */ + int ePatchset, /* Use the patchset format if true */ SessionTable *pTab, /* Table object to append header for */ int *pRc /* IN/OUT: Error code */ ){ /* Write a table header */ - sessionAppendByte(pBuf, (bPatchset ? 'P' : 'T'), pRc); + sessionAppendByte(pBuf, (ePatchset==SESSIONS_PATCHSET) ? 'P' : 'T', pRc); sessionAppendVarint(pBuf, pTab->nCol, pRc); sessionAppendBlob(pBuf, pTab->abPK, pTab->nCol, pRc); sessionAppendBlob(pBuf, (u8 *)pTab->zName, (int)strlen(pTab->zName)+1, pRc); @@ -2225,7 +2232,7 @@ static void sessionAppendTableHdr( */ static int sessionGenerateChangeset( sqlite3_session *pSession, /* Session object */ - int bPatchset, /* True for patchset, false for changeset */ + int ePatchset, /* One of SESSIONS_CHANGESET etc. */ int (*xOutput)(void *pOut, const void *pData, int nData), void *pOut, /* First argument for xOutput */ int *pnChangeset, /* OUT: Size of buffer at *ppChangeset */ @@ -2270,7 +2277,7 @@ static int sessionGenerateChangeset( } /* Write a table header */ - sessionAppendTableHdr(&buf, bPatchset, pTab, &rc); + sessionAppendTableHdr(&buf, ePatchset, pTab, &rc); /* Build and compile a statement to execute: */ if( rc==SQLITE_OK ){ @@ -2294,10 +2301,10 @@ static int sessionGenerateChangeset( sessionAppendCol(&buf, pSel, iCol, &rc); } }else{ - rc = sessionAppendUpdate(&buf, bPatchset, pSel, p, abPK); + rc = sessionAppendUpdate(&buf, ePatchset, pSel, p, abPK); } }else if( p->op!=SQLITE_INSERT ){ - rc = sessionAppendDelete(&buf, bPatchset, p, nCol, abPK); + rc = sessionAppendDelete(&buf, ePatchset, p, nCol, abPK); } if( rc==SQLITE_OK ){ rc = sqlite3_reset(pSel); @@ -2354,7 +2361,8 @@ int sqlite3session_changeset( int *pnChangeset, /* OUT: Size of buffer at *ppChangeset */ void **ppChangeset /* OUT: Buffer containing changeset */ ){ - return sessionGenerateChangeset(pSession, 0, 0, 0, pnChangeset, ppChangeset); + return sessionGenerateChangeset( + pSession, SESSIONS_CHANGESET, 0, 0, pnChangeset, ppChangeset); } /* @@ -2365,7 +2373,8 @@ int sqlite3session_changeset_strm( int (*xOutput)(void *pOut, const void *pData, int nData), void *pOut ){ - return sessionGenerateChangeset(pSession, 0, xOutput, pOut, 0, 0); + return sessionGenerateChangeset( + pSession, SESSIONS_CHANGESET, xOutput, pOut, 0, 0); } /* @@ -2376,7 +2385,8 @@ int sqlite3session_patchset_strm( int (*xOutput)(void *pOut, const void *pData, int nData), void *pOut ){ - return sessionGenerateChangeset(pSession, 1, xOutput, pOut, 0, 0); + return sessionGenerateChangeset( + pSession, SESSIONS_PATCHSET, xOutput, pOut, 0, 0); } /* @@ -2391,9 +2401,20 @@ int sqlite3session_patchset( int *pnPatchset, /* OUT: Size of buffer at *ppChangeset */ void **ppPatchset /* OUT: Buffer containing changeset */ ){ - return sessionGenerateChangeset(pSession, 1, 0, 0, pnPatchset, ppPatchset); + return sessionGenerateChangeset( + pSession, SESSIONS_PATCHSET, 0, 0, pnPatchset, ppPatchset); } +int sqlite3session_fullchangeset( + sqlite3_session *pSession, /* Session object */ + int *pnChangeset, /* OUT: Size of buffer at *ppChangeset */ + void **ppChangeset /* OUT: Buffer containing changeset */ +){ + return sessionGenerateChangeset( + pSession, SESSIONS_FULLCHANGESET, 0, 0, pnChangeset, ppChangeset); +} + + /* ** Enable or disable the session object passed as the first argument. */ @@ -4463,10 +4484,11 @@ static int sessionChangegroupOutput( ** hash tables attached to the SessionTable objects in list p->pList. */ for(pTab=pGrp->pList; rc==SQLITE_OK && pTab; pTab=pTab->pNext){ + int eChangeset = pGrp->bPatch ? SESSIONS_PATCHSET : SESSIONS_CHANGESET; int i; if( pTab->nEntry==0 ) continue; - sessionAppendTableHdr(&buf, pGrp->bPatch, pTab, &rc); + sessionAppendTableHdr(&buf, eChangeset, pTab, &rc); for(i=0; inChange; i++){ SessionChange *p; for(p=pTab->apChange[i]; p; p=p->pNext){ diff --git a/ext/session/sqlite3session.h b/ext/session/sqlite3session.h index 23d9a33aef..91ba511936 100644 --- a/ext/session/sqlite3session.h +++ b/ext/session/sqlite3session.h @@ -281,6 +281,19 @@ int sqlite3session_changeset( void **ppChangeset /* OUT: Buffer containing changeset */ ); +/* +** CAPI3REF: Generate A Full Changeset From A Session Object +** +** This function is similar to sqlite3session_changeset(), except that for +** each row affected by an UPDATE statement, all old.* values are recorded +** as part of the changeset, not just those modified. +*/ +int sqlite3session_fullchangeset( + sqlite3_session *pSession, /* Session object */ + int *pnChangeset, /* OUT: Size of buffer at *ppChangeset */ + void **ppChangeset /* OUT: Buffer containing changeset */ +); + /* ** CAPI3REF: Load The Difference Between Tables Into A Session ** diff --git a/ext/session/test_session.c b/ext/session/test_session.c index 3f4cdd3c34..02274c7f84 100644 --- a/ext/session/test_session.c +++ b/ext/session/test_session.c @@ -113,6 +113,7 @@ static int testStreamOutput( ** $session indirect INTEGER ** $session patchset ** $session table_filter SCRIPT +** $session fullchangeset */ static int SQLITE_TCLAPI test_session_cmd( void *clientData, @@ -126,17 +127,17 @@ static int SQLITE_TCLAPI test_session_cmd( const char *zSub; int nArg; const char *zMsg; - int iSub; } aSub[] = { - { "attach", 1, "TABLE", }, /* 0 */ - { "changeset", 0, "", }, /* 1 */ - { "delete", 0, "", }, /* 2 */ - { "enable", 1, "BOOL", }, /* 3 */ - { "indirect", 1, "BOOL", }, /* 4 */ - { "isempty", 0, "", }, /* 5 */ - { "table_filter", 1, "SCRIPT", }, /* 6 */ + { "attach", 1, "TABLE" }, /* 0 */ + { "changeset", 0, "" }, /* 1 */ + { "delete", 0, "" }, /* 2 */ + { "enable", 1, "BOOL" }, /* 3 */ + { "indirect", 1, "BOOL" }, /* 4 */ + { "isempty", 0, "" }, /* 5 */ + { "table_filter", 1, "SCRIPT" }, /* 6 */ { "patchset", 0, "", }, /* 7 */ - { "diff", 2, "FROMDB TBL", }, /* 8 */ + { "diff", 2, "FROMDB TBL" }, /* 8 */ + { "fullchangeset",0, "" }, /* 9 */ { 0 } }; int iSub; @@ -166,10 +167,11 @@ static int SQLITE_TCLAPI test_session_cmd( break; } + case 9: /* fullchangeset */ case 7: /* patchset */ case 1: { /* changeset */ TestSessionsBlob o = {0, 0}; - if( test_tcl_integer(interp, SESSION_STREAM_TCL_VAR) ){ + if( iSub!=9 && test_tcl_integer(interp, SESSION_STREAM_TCL_VAR) ){ void *pCtx = (void*)&o; if( iSub==7 ){ rc = sqlite3session_patchset_strm(pSession, testStreamOutput, pCtx); @@ -179,6 +181,8 @@ static int SQLITE_TCLAPI test_session_cmd( }else{ if( iSub==7 ){ rc = sqlite3session_patchset(pSession, &o.n, &o.p); + }else if( iSub==9 ){ + rc = sqlite3session_fullchangeset(pSession, &o.n, &o.p); }else{ rc = sqlite3session_changeset(pSession, &o.n, &o.p); } @@ -193,6 +197,7 @@ static int SQLITE_TCLAPI test_session_cmd( break; } + case 2: /* delete */ Tcl_DeleteCommand(interp, Tcl_GetString(objv[0])); break; diff --git a/manifest b/manifest index d16bdf85a5..8e6e0945fc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\scouple\sof\sextra\stests\sto\schangebatch1.test. -D 2016-08-22T21:01:39.413 +C Add\sa\snew\smethod\sto\ssessions\s-\ssqlite3sessions_fullchangeset()\s-\sto\sreturn\sa\schangeset\sthat\salways\scontains\svalues\sfor\sall\sold.*\sfields.\sUpdate\schangebatch\sto\suse\sthese\svalues\sto\smore\sreliably\sdetect\smulti-column\sUNIQUE\sconstraint\sviolations. +D 2016-08-23T17:02:28.920 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc d66d0395c38571aab3804f8db0fa20707ae4609a @@ -281,7 +281,7 @@ F ext/rtree/rtree_util.tcl 06aab2ed5b826545bf215fff90ecb9255a8647ea F ext/rtree/sqlite3rtree.h 9c5777af3d2921c7b4ae4954e8e5697502289d28 F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 -F ext/session/changebatch1.test b2d1a8c8b8f86881f50b481eae233f8abfb61436 +F ext/session/changebatch1.test f3e5462189ebe238b57ddf31f17e5f6cb410f895 F ext/session/changeset.c 4ccbaa4531944c24584bf6a61ba3a39c62b6267a F ext/session/session1.test 98f384736e2bc21ccf5ed81bdadcff4ad863393b F ext/session/session2.test 284de45abae4cc1082bc52012ee81521d5ac58e0 @@ -301,11 +301,11 @@ F ext/session/sessionG.test 01ef705096a9d3984eebdcca79807a211dee1b60 F ext/session/session_common.tcl a1293167d14774b5e728836720497f40fe4ea596 F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7 F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0 -F ext/session/sqlite3changebatch.c 37fb1c87d7b3ccaff194411ff8344d9cc056bd98 +F ext/session/sqlite3changebatch.c 7ddd1b44422508306c50c37056bb13d5e0492bd0 F ext/session/sqlite3changebatch.h 50a302e4fc535324309607b13a1993bca074758b -F ext/session/sqlite3session.c 37485891b4add26cf61495df193c419f36556a32 -F ext/session/sqlite3session.h 69bf73cfd71e58f2ae5d2aa935b2c1a541aee555 -F ext/session/test_session.c 24968972a5709d2ccf5d570f1a13ce009fbc0d86 +F ext/session/sqlite3session.c e5591f76aea6058720e04f78ae9e88487eb56c6b +F ext/session/sqlite3session.h c772b5440f41af44631891aa7f352e9a44b740ad +F ext/session/test_session.c 9e6a4313dc94b053edd33f54c3ffc053aeddff45 F ext/userauth/sqlite3userauth.h 19cb6f0e31316d0ee4afdfb7a85ef9da3333a220 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e @@ -1514,7 +1514,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0c9fd6b723041955b5182caa430312e5124fdc83 -R 0bf0e85497e377afe1f85fb7abd8225d +P 207d970b7956c38af42c389b91a741a68b2c4eec +R fcf84064b854d920af796eb5c195b948 U dan -Z cf85cb19b321088cfc5e96ee875ee34c +Z e2f66a65ddf5985dbdfd723a07f7b7cc diff --git a/manifest.uuid b/manifest.uuid index f8bc749f9d..cecdca57cb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -207d970b7956c38af42c389b91a741a68b2c4eec \ No newline at end of file +efa761b2f509844b9212dd20bf0d082c6338e83f \ No newline at end of file From 7d931b985a5d04c42a4b92002bef752c5da787ff Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 23 Aug 2016 18:09:37 +0000 Subject: [PATCH 040/179] Fix harmless compiler warnings in changebatch. FossilOrigin-Name: a721a738184d914fcde3f5684099984a9373dff3 --- ext/session/sqlite3changebatch.c | 4 ++-- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ext/session/sqlite3changebatch.c b/ext/session/sqlite3changebatch.c index 5672a574c5..186bcab99c 100644 --- a/ext/session/sqlite3changebatch.c +++ b/ext/session/sqlite3changebatch.c @@ -79,7 +79,7 @@ static int cbHash(sqlite3_changebatch *p, BatchIndexEntry *pEntry){ unsigned char *pEnd = (unsigned char*)&pEntry->aRecord[pEntry->szRecord]; unsigned char *pIter; - for(pIter=pEntry->aRecord; pIteraRecord; pIternHash = nNew; for(i=0; iapHash[i] ){ + while( (pEntry=p->apHash[i])!=0 ){ int iHash = cbHash(p, pEntry); p->apHash[i] = pEntry->pNext; pEntry->pNext = apNew[iHash]; diff --git a/manifest b/manifest index 8e6e0945fc..4b3e6de352 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\snew\smethod\sto\ssessions\s-\ssqlite3sessions_fullchangeset()\s-\sto\sreturn\sa\schangeset\sthat\salways\scontains\svalues\sfor\sall\sold.*\sfields.\sUpdate\schangebatch\sto\suse\sthese\svalues\sto\smore\sreliably\sdetect\smulti-column\sUNIQUE\sconstraint\sviolations. -D 2016-08-23T17:02:28.920 +C Fix\sharmless\scompiler\swarnings\sin\schangebatch. +D 2016-08-23T18:09:37.916 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc d66d0395c38571aab3804f8db0fa20707ae4609a @@ -301,7 +301,7 @@ F ext/session/sessionG.test 01ef705096a9d3984eebdcca79807a211dee1b60 F ext/session/session_common.tcl a1293167d14774b5e728836720497f40fe4ea596 F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7 F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0 -F ext/session/sqlite3changebatch.c 7ddd1b44422508306c50c37056bb13d5e0492bd0 +F ext/session/sqlite3changebatch.c 72ec2f5c70af4e9e938222a98962bc05b82e9bf4 F ext/session/sqlite3changebatch.h 50a302e4fc535324309607b13a1993bca074758b F ext/session/sqlite3session.c e5591f76aea6058720e04f78ae9e88487eb56c6b F ext/session/sqlite3session.h c772b5440f41af44631891aa7f352e9a44b740ad @@ -1514,7 +1514,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 207d970b7956c38af42c389b91a741a68b2c4eec -R fcf84064b854d920af796eb5c195b948 -U dan -Z e2f66a65ddf5985dbdfd723a07f7b7cc +P efa761b2f509844b9212dd20bf0d082c6338e83f +R 7f5bf9af3be1c169a39e0edb328bafbb +U drh +Z e5dc9f2e44b6adaa79d5e6bc2fc1ab5e diff --git a/manifest.uuid b/manifest.uuid index cecdca57cb..ad7a6d4e5d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -efa761b2f509844b9212dd20bf0d082c6338e83f \ No newline at end of file +a721a738184d914fcde3f5684099984a9373dff3 \ No newline at end of file From a8dee8df646b19de71ed288f088f3439209dfe1d Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 23 Aug 2016 19:02:21 +0000 Subject: [PATCH 041/179] Add further tests for changebatch. And a fix to prevent a changeset from conflicting with itself. FossilOrigin-Name: 506d6ff4b64c72d4ca65f0d15e1fdf8a132556bc --- ext/session/changebatch1.test | 73 +++++++++++++++++-- ext/session/changebatchfault.test | 42 +++++++++++ ext/session/sqlite3changebatch.c | 114 ++++++++++++++++++------------ manifest | 17 ++--- manifest.uuid | 2 +- 5 files changed, 189 insertions(+), 59 deletions(-) create mode 100644 ext/session/changebatchfault.test diff --git a/ext/session/changebatch1.test b/ext/session/changebatch1.test index fc3f50ab4c..2fbe368947 100644 --- a/ext/session/changebatch1.test +++ b/ext/session/changebatch1.test @@ -19,6 +19,7 @@ ifcapable !session {finish_test; return} set testprefix changebatch1 + proc sql_to_changeset {method sql} { sqlite3session S db main S attach * @@ -37,13 +38,13 @@ proc do_changebatch_test {tn method args} { sqlite3changebatch cb db set i 1 foreach ::cs [lrange $C 0 end-1] { - do_test $tn.$i { cb add [set ::cs] } SQLITE_OK + set rc [cb add $::cs] + if {$rc!="SQLITE_OK"} { error "expected SQLITE_OK, got $rc (i=$i)" } incr i } set ::cs [lindex $C end] - do_test $tn.$i { cb add [set ::cs] } SQLITE_CONSTRAINT - + do_test $tn { cb add [set ::cs] } SQLITE_CONSTRAINT cb delete } @@ -94,6 +95,7 @@ foreach {tn testfunction} { do_execsql_test $tn.2.0 { CREATE TABLE x1(a, b PRIMARY KEY, c UNIQUE); CREATE TABLE x2(a PRIMARY KEY, b UNIQUE, c UNIQUE); + CREATE INDEX x1a ON x1(a); INSERT INTO x1 VALUES(1, 1, 'a'); INSERT INTO x1 VALUES(1, 2, 'b'); @@ -111,6 +113,53 @@ foreach {tn testfunction} { } { INSERT INTO x1 VALUES(1, 5, 'a'); } + + set L [list] + for {set i 1000} {$i < 10000} {incr i} { + lappend L "INSERT INTO x2 VALUES($i, $i, 'x' || $i)" + } + lappend L "DELETE FROM x2 WHERE b=1005" + $testfunction $tn.2.3 {*}$L + + execsql { INSERT INTO x1 VALUES('f', 'f', 'f') } + $testfunction $tn.2.4 { + INSERT INTO x2 VALUES('f', 'f', 'f'); + } { + INSERT INTO x1 VALUES('g', 'g', 'g'); + } { + DELETE FROM x1 WHERE b='f'; + } { + INSERT INTO x2 VALUES('g', 'g', 'g'); + } { + INSERT INTO x1 VALUES('f', 'f', 'f'); + } + + execsql { + DELETE FROM x1; + INSERT INTO x1 VALUES(1.5, 1.5, 1.5); + } + $testfunction $tn.2.5 { + DELETE FROM x1 WHERE b BETWEEN 1 AND 2; + } { + INSERT INTO x1 VALUES(2.5, 2.5, 2.5); + } { + INSERT INTO x1 VALUES(1.5, 1.5, 1.5); + } + + execsql { + DELETE FROM x2; + INSERT INTO x2 VALUES(X'abcd', X'1234', X'7890'); + INSERT INTO x2 VALUES(X'0000', X'0000', X'0000'); + } + breakpoint + $testfunction $tn.2.6 { + UPDATE x2 SET c = X'1234' WHERE a=X'abcd'; + INSERT INTO x2 VALUES(X'1234', X'abcd', X'7890'); + } { + DELETE FROM x2 WHERE b=X'0000'; + } { + INSERT INTO x2 VALUES(1, X'0000', NULL); + } } #------------------------------------------------------------------------- @@ -151,7 +200,23 @@ do_test 3.1.2 { cb delete } {} +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.0 { + CREATE TABLE t1(x, y, z, PRIMARY KEY(x, y), UNIQUE(z)); +} + +do_test 4.1 { + set c1 [sql_to_changeset fullchangeset { INSERT INTO t1 VALUES(1, 2, 3) }] + execsql { + DROP TABLE t1; + CREATE TABLE t1(w, x, y, z, PRIMARY KEY(x, y), UNIQUE(z)); + } + sqlite3changebatch cb db + list [catch { cb add $c1 } msg] $msg +} {1 SQLITE_RANGE} -finish_test +finish_test diff --git a/ext/session/changebatchfault.test b/ext/session/changebatchfault.test new file mode 100644 index 0000000000..209b60e0c8 --- /dev/null +++ b/ext/session/changebatchfault.test @@ -0,0 +1,42 @@ +# 2011 Mar 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the session module. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} +set testprefix changebatchfault + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c PRIMARY KEY, UNIQUE(a, b)); + INSERT INTO t1 VALUES('a', 'a', 'a'); + INSERT INTO t1 VALUES('b', 'b', 'b'); +} + +set ::c1 [changeset_from_sql { delete from t1 where c='a' }] +set ::c2 [changeset_from_sql { insert into t1 values('c', 'c', 'c') }] + +do_faultsim_test 1 -faults oom-* -body { + sqlite3changebatch cb db + cb add $::c1 + cb add $::c2 +} -test { + faultsim_test_result {0 SQLITE_OK} {1 SQLITE_NOMEM} + catch { cb delete } +} + + +finish_test diff --git a/ext/session/sqlite3changebatch.c b/ext/session/sqlite3changebatch.c index 186bcab99c..a31d04ff4e 100644 --- a/ext/session/sqlite3changebatch.c +++ b/ext/session/sqlite3changebatch.c @@ -154,10 +154,8 @@ static int cbAddIndex( } if( rc==SQLITE_OK ){ - int rc2; while( SQLITE_ROW==sqlite3_step(pIndexInfo) ){ nCol++; } - rc2 = sqlite3_reset(pIndexInfo); - if( rc==SQLITE_OK ) rc = rc2; + rc = sqlite3_reset(pIndexInfo); } pNew = (BatchIndex*)cbMalloc(&rc, sizeof(BatchIndex) + sizeof(int) * nCol); @@ -172,8 +170,7 @@ static int cbAddIndex( int j = sqlite3_column_int(pIndexInfo, 1); pNew->aiCol[i] = j; } - rc2 = sqlite3_reset(pIndexInfo); - if( rc==SQLITE_OK ) rc = rc2; + rc = sqlite3_reset(pIndexInfo); } if( rc==SQLITE_OK ){ @@ -187,6 +184,19 @@ static int cbAddIndex( return rc; } +/* +** Free the object passed as the first argument. +*/ +static void cbFreeTable(BatchTable *pTab){ + BatchIndex *pIdx; + BatchIndex *pIdxNext; + for(pIdx=pTab->pIdx; pIdx; pIdx=pIdxNext){ + pIdxNext = pIdx->pNext; + cbFree(pIdx); + } + cbFree(pTab); +} + /* ** Find or create the BatchTable object named zTab. */ @@ -232,6 +242,9 @@ static int cbFindTable( if( rc==SQLITE_OK ){ pRet->pNext = p->pTab; p->pTab = pRet; + }else{ + cbFreeTable(pRet); + pRet = 0; } } } @@ -240,6 +253,16 @@ static int cbFindTable( return rc; } +/* +** Extract value iVal from the changeset iterator passed as the first +** argument. Set *ppVal to point to the value before returning. +** +** This function attempts to extract the value using function xVal +** (which is always either sqlite3changeset_new or sqlite3changeset_old). +** If the call returns SQLITE_OK but does not supply an sqlite3_value* +** pointer, an attempt to extract the value is made using the xFallback +** function. +*/ static int cbGetChangesetValue( sqlite3_changeset_iter *pIter, int (*xVal)(sqlite3_changeset_iter*,int,sqlite3_value**), @@ -300,33 +323,33 @@ static int cbAddToHash( pNew->iIdxId = pIdx->iId; pNew->szRecord = sz; - for(i=0; rc==SQLITE_OK && inCol; i++){ + for(i=0; inCol; i++){ + int eType; sqlite3_value *pVal; rc = cbGetChangesetValue(pIter, xVal, xFallback, pIdx->aiCol[i], &pVal); - if( rc==SQLITE_OK ){ - int eType = sqlite3_value_type(pVal); - pNew->aRecord[iOut++] = eType; - switch( eType ){ - case SQLITE_INTEGER: { - sqlite3_int64 i64 = sqlite3_value_int64(pVal); - memcpy(&pNew->aRecord[iOut], &i64, 8); - iOut += 8; - break; - } - case SQLITE_FLOAT: { - double d64 = sqlite3_value_double(pVal); - memcpy(&pNew->aRecord[iOut], &d64, sizeof(double)); - iOut += sizeof(double); - break; - } + if( rc!=SQLITE_OK ) break; /* coverage: condition is never true */ + eType = sqlite3_value_type(pVal); + pNew->aRecord[iOut++] = eType; + switch( eType ){ + case SQLITE_INTEGER: { + sqlite3_int64 i64 = sqlite3_value_int64(pVal); + memcpy(&pNew->aRecord[iOut], &i64, 8); + iOut += 8; + break; + } + case SQLITE_FLOAT: { + double d64 = sqlite3_value_double(pVal); + memcpy(&pNew->aRecord[iOut], &d64, sizeof(double)); + iOut += sizeof(double); + break; + } - default: { - int nByte = sqlite3_value_bytes(pVal); - const char *z = (const char*)sqlite3_value_blob(pVal); - memcpy(&pNew->aRecord[iOut], z, nByte); - iOut += nByte; - break; - } + default: { + int nByte = sqlite3_value_bytes(pVal); + const char *z = (const char*)sqlite3_value_blob(pVal); + memcpy(&pNew->aRecord[iOut], z, nByte); + iOut += nByte; + break; } } } @@ -359,9 +382,10 @@ static int cbAddToHash( p->apHash[iHash] = pNew; p->nEntry++; } + }else{ + cbFree(pNew); } - p->iChangesetId++; return rc; } @@ -391,15 +415,18 @@ int sqlite3changebatch_add(sqlite3_changebatch *p, void *pBuf, int nBuf){ assert( op==SQLITE_INSERT || op==SQLITE_UPDATE || op==SQLITE_DELETE ); rc = cbFindTable(p, zTab, &pTab); - for(pIdx=pTab->pIdx; pIdx && rc==SQLITE_OK; pIdx=pIdx->pNext){ - if( op==SQLITE_UPDATE && pIdx->bPk ) continue; - if( op==SQLITE_UPDATE || op==SQLITE_DELETE ){ - rc = cbAddToHash(p, pIter, pIdx, sqlite3changeset_old, 0, &bConf); - } - if( op==SQLITE_UPDATE || op==SQLITE_INSERT ){ - rc = cbAddToHash(p, pIter, pIdx, - sqlite3changeset_new, sqlite3changeset_old, &bConf - ); + assert( pTab || rc!=SQLITE_OK ); + if( pTab ){ + for(pIdx=pTab->pIdx; pIdx && rc==SQLITE_OK; pIdx=pIdx->pNext){ + if( op==SQLITE_UPDATE && pIdx->bPk ) continue; + if( op==SQLITE_UPDATE || op==SQLITE_DELETE ){ + rc = cbAddToHash(p, pIter, pIdx, sqlite3changeset_old, 0, &bConf); + } + if( op==SQLITE_UPDATE || op==SQLITE_INSERT ){ + rc = cbAddToHash(p, pIter, pIdx, + sqlite3changeset_new, sqlite3changeset_old, &bConf + ); + } } } if( rc!=SQLITE_OK ) break; @@ -412,6 +439,7 @@ int sqlite3changebatch_add(sqlite3_changebatch *p, void *pBuf, int nBuf){ if( rc==SQLITE_OK && bConf ){ rc = SQLITE_CONSTRAINT; } + p->iChangesetId++; return rc; } @@ -442,14 +470,8 @@ void sqlite3changebatch_delete(sqlite3_changebatch *p){ sqlite3changebatch_zero(p); for(pTab=p->pTab; pTab; pTab=pTabNext){ - BatchIndex *pIdx; - BatchIndex *pIdxNext; - for(pIdx=pTab->pIdx; pIdx; pIdx=pIdxNext){ - pIdxNext = pIdx->pNext; - cbFree(pIdx); - } pTabNext = pTab->pNext; - cbFree(pTab); + cbFreeTable(pTab); } cbFree(p); } diff --git a/manifest b/manifest index 4b3e6de352..3cea0f9b42 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sin\schangebatch. -D 2016-08-23T18:09:37.916 +C Add\sfurther\stests\sfor\schangebatch.\sAnd\sa\sfix\sto\sprevent\sa\schangeset\sfrom\sconflicting\swith\sitself. +D 2016-08-23T19:02:21.914 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc d66d0395c38571aab3804f8db0fa20707ae4609a @@ -281,7 +281,8 @@ F ext/rtree/rtree_util.tcl 06aab2ed5b826545bf215fff90ecb9255a8647ea F ext/rtree/sqlite3rtree.h 9c5777af3d2921c7b4ae4954e8e5697502289d28 F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 -F ext/session/changebatch1.test f3e5462189ebe238b57ddf31f17e5f6cb410f895 +F ext/session/changebatch1.test 9ceaa8f7b2a505933e250fbe6cbc550e4ce1e59d +F ext/session/changebatchfault.test be49c793219bf387ad692a60856b921f0854ad6d F ext/session/changeset.c 4ccbaa4531944c24584bf6a61ba3a39c62b6267a F ext/session/session1.test 98f384736e2bc21ccf5ed81bdadcff4ad863393b F ext/session/session2.test 284de45abae4cc1082bc52012ee81521d5ac58e0 @@ -301,7 +302,7 @@ F ext/session/sessionG.test 01ef705096a9d3984eebdcca79807a211dee1b60 F ext/session/session_common.tcl a1293167d14774b5e728836720497f40fe4ea596 F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7 F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0 -F ext/session/sqlite3changebatch.c 72ec2f5c70af4e9e938222a98962bc05b82e9bf4 +F ext/session/sqlite3changebatch.c 4b4fe1d52587e5fdb28930466409712f0e4b619d F ext/session/sqlite3changebatch.h 50a302e4fc535324309607b13a1993bca074758b F ext/session/sqlite3session.c e5591f76aea6058720e04f78ae9e88487eb56c6b F ext/session/sqlite3session.h c772b5440f41af44631891aa7f352e9a44b740ad @@ -1514,7 +1515,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P efa761b2f509844b9212dd20bf0d082c6338e83f -R 7f5bf9af3be1c169a39e0edb328bafbb -U drh -Z e5dc9f2e44b6adaa79d5e6bc2fc1ab5e +P a721a738184d914fcde3f5684099984a9373dff3 +R b3c5ba41110022813f99ffae5c67351c +U dan +Z 78075b3679b308d5ec26925a2eace0fc diff --git a/manifest.uuid b/manifest.uuid index ad7a6d4e5d..96566a509e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a721a738184d914fcde3f5684099984a9373dff3 \ No newline at end of file +506d6ff4b64c72d4ca65f0d15e1fdf8a132556bc \ No newline at end of file From ff8e42e2ca85961bb8e45ee6fe52bba4c66e5b81 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 24 Aug 2016 19:14:59 +0000 Subject: [PATCH 042/179] Add the sqlite3changebatch_db() API. FossilOrigin-Name: bee44ebc532f37e3fe61c18878e0d3db06805190 --- ext/session/sqlite3changebatch.c | 7 +++++++ ext/session/sqlite3changebatch.h | 7 +++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/ext/session/sqlite3changebatch.c b/ext/session/sqlite3changebatch.c index a31d04ff4e..0490d5f977 100644 --- a/ext/session/sqlite3changebatch.c +++ b/ext/session/sqlite3changebatch.c @@ -476,4 +476,11 @@ void sqlite3changebatch_delete(sqlite3_changebatch *p){ cbFree(p); } +/* +** Return the db handle. +*/ +sqlite3 *sqlite3changebatch_db(sqlite3_changebatch *p){ + return p->db; +} + #endif /* SQLITE_ENABLE_SESSION && SQLITE_ENABLE_PREUPDATE_HOOK */ diff --git a/ext/session/sqlite3changebatch.h b/ext/session/sqlite3changebatch.h index 9afa6c174e..3476629e58 100644 --- a/ext/session/sqlite3changebatch.h +++ b/ext/session/sqlite3changebatch.h @@ -66,6 +66,13 @@ int sqlite3changebatch_add(sqlite3_changebatch*, void *p, int n); */ void sqlite3changebatch_zero(sqlite3_changebatch*); +/* +** Return a copy of the first argument passed to the sqlite3changebatch_new() +** call used to create the changebatch object passed as the only argument +** to this function. +*/ +sqlite3 *sqlite3changebatch_db(sqlite3_changebatch*); + /* ** Delete a changebatch object. */ diff --git a/manifest b/manifest index 3cea0f9b42..1a738945f8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sfurther\stests\sfor\schangebatch.\sAnd\sa\sfix\sto\sprevent\sa\schangeset\sfrom\sconflicting\swith\sitself. -D 2016-08-23T19:02:21.914 +C Add\sthe\ssqlite3changebatch_db()\sAPI. +D 2016-08-24T19:14:59.159 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc d66d0395c38571aab3804f8db0fa20707ae4609a @@ -302,8 +302,8 @@ F ext/session/sessionG.test 01ef705096a9d3984eebdcca79807a211dee1b60 F ext/session/session_common.tcl a1293167d14774b5e728836720497f40fe4ea596 F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7 F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0 -F ext/session/sqlite3changebatch.c 4b4fe1d52587e5fdb28930466409712f0e4b619d -F ext/session/sqlite3changebatch.h 50a302e4fc535324309607b13a1993bca074758b +F ext/session/sqlite3changebatch.c 96011bdf72ac12ebf707f4d0f0e719e333631605 +F ext/session/sqlite3changebatch.h e72016998c9a22d439ddfd547b69e1ebac810c24 F ext/session/sqlite3session.c e5591f76aea6058720e04f78ae9e88487eb56c6b F ext/session/sqlite3session.h c772b5440f41af44631891aa7f352e9a44b740ad F ext/session/test_session.c 9e6a4313dc94b053edd33f54c3ffc053aeddff45 @@ -1515,7 +1515,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a721a738184d914fcde3f5684099984a9373dff3 -R b3c5ba41110022813f99ffae5c67351c +P 506d6ff4b64c72d4ca65f0d15e1fdf8a132556bc +R 437a725ec959b210c45a517f0a4720a0 U dan -Z 78075b3679b308d5ec26925a2eace0fc +Z 7a3026050d391945a1ada937b36aa9ce diff --git a/manifest.uuid b/manifest.uuid index 96566a509e..08b140baf5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -506d6ff4b64c72d4ca65f0d15e1fdf8a132556bc \ No newline at end of file +bee44ebc532f37e3fe61c18878e0d3db06805190 \ No newline at end of file From 0195d8ef5d7e3dd32819178a567c3a32f2d22698 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 9 Jan 2017 06:53:00 +0000 Subject: [PATCH 043/179] Disable the optimization from [8cb8516d] (omit TableLock instructions for non-sharable databases) on this branch. This branch uses the TableLock instructions to ensure that the db schema is not written from within a BEGIN CONCURRENT transaction. FossilOrigin-Name: c8ca3e0a8d82dbb077551c2d952cb2043f78333b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/build.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index d78d211a93..38fa1464ab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Upgrade\sthis\sbranch\sto\s3.16\splus\sthe\svarious\sfixes\sthat\sappeared\safter\sits\nrelease. -D 2017-01-09T06:33:50.719 +C Disable\sthe\soptimization\sfrom\s[8cb8516d]\s(omit\sTableLock\sinstructions\sfor\nnon-sharable\sdatabases)\son\sthis\sbranch.\sThis\sbranch\suses\sthe\sTableLock\ninstructions\sto\sensure\sthat\sthe\sdb\sschema\sis\snot\swritten\sfrom\swithin\sa\sBEGIN\nCONCURRENT\stransaction. +D 2017-01-09T06:53:00.359 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -334,7 +334,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca F src/btree.c 5e5bf5079aae45b91e71eb087f3eddd28deafafe F src/btree.h 9306cfe42eed20cf2d0e407172704f5193af26cc F src/btreeInt.h bade42398d4dcadd2263c6d17886e812e2471e87 -F src/build.c efef69bbc7d0729a9b1e6d71f1047d0c5e1cdb5b +F src/build.c 58bd67d6648d6120592605768a7008fd52fb2656 F src/callback.c 2e76147783386374bf01b227f752c81ec872d730 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 9f2296a4e5d26ebf0e0d95a0af4628f1ea694e7a @@ -1548,7 +1548,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d9f8918c5b7b6c8540b3f433142e1b4aa4433885 6696cd1878be4bd44a24841b04163e52d847711e -R 47f5758f699f0ce7270fcc21bb40e26e +P d0e212d08f82dfb5e42a156b3e2bb03dd8e21258 +R e3f8f4a25fa664daafdf2b1037a6c994 U dan -Z 398299ab95e9ac138577a5fdad763428 +Z 5b5ce2ca54ddaa75cd9b42bf54bd493e diff --git a/manifest.uuid b/manifest.uuid index a10503843d..01c64fc58f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d0e212d08f82dfb5e42a156b3e2bb03dd8e21258 \ No newline at end of file +c8ca3e0a8d82dbb077551c2d952cb2043f78333b \ No newline at end of file diff --git a/src/build.c b/src/build.c index e412a44bc1..ea282ee8ea 100644 --- a/src/build.c +++ b/src/build.c @@ -59,8 +59,10 @@ void sqlite3TableLock( TableLock *p; assert( iDb>=0 ); +#ifdef SQLITE_OMIT_CONCURRENT if( iDb==1 ) return; if( !sqlite3BtreeSharable(pParse->db->aDb[iDb].pBt) ) return; +#endif for(i=0; inTableLock; i++){ p = &pToplevel->aTableLock[i]; if( p->iDb==iDb && p->iTab==iTab ){ From fe329a737060c6e9aea2e41f6e3661425b75aa35 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 28 Mar 2017 18:48:43 +0000 Subject: [PATCH 044/179] Version 3.18.0 Release Candidate FossilOrigin-Name: 424a0d380332858ee55bdebc4af3789f74e70a2b3ba1cf29d84b9b4bcf3e2e37 --- manifest | 11 +++++++---- manifest.uuid | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/manifest b/manifest index 65dae1b276..ca56437574 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\snew\ssqlite3_set_last_insert_rowid()\sinterface\sto\sthe\sextension\nloader\sthunk. -D 2017-03-25T19:16:41.259 +C Version\s3.18.0\sRelease\sCandidate +D 2017-03-28T18:48:43.315 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -1569,7 +1569,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 55df410b2cd27dc4c7670bcc1196fa3a0e9e601fc24c42e53d6b0a26ee3e4b45 +P 8469fc0d48d6af0accef9b8a84e08ad2ca32351907510d177b4ca4815c1ea7cb R e1b145e92e118b2bc2de7100fb7561b5 +T *branch * version-3.18 +T *sym-version-3.18 * +T -sym-trunk * U drh -Z e401f4d2ca1590406e5405bb5cd2850e +Z 2998f3cf8a15fbd1f3dc49f1b039f0d5 diff --git a/manifest.uuid b/manifest.uuid index 6ed302dbe3..5619e48c7b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8469fc0d48d6af0accef9b8a84e08ad2ca32351907510d177b4ca4815c1ea7cb \ No newline at end of file +424a0d380332858ee55bdebc4af3789f74e70a2b3ba1cf29d84b9b4bcf3e2e37 \ No newline at end of file From 6dbb452b381e7292f0b20739456e7652277a6fb9 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 18 May 2017 20:47:31 +0000 Subject: [PATCH 045/179] Fix a problem on this branch causing some page-level read-locks to be omitted. FossilOrigin-Name: 0eed152162b4721f7aaba8b480426476978a869e9578f100fca7b1d32942fe1a --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/pager.c | 1 + test/concurrent.test | 30 ++++++++++++++++++++++++++++++ 4 files changed, 40 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 9f8c231968..cf8e9ad002 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\schanges\sfrom\strunk. -D 2017-05-15T17:34:23.124 +C Fix\sa\sproblem\son\sthis\sbranch\scausing\ssome\spage-level\sread-locks\sto\sbe\somitted. +D 2017-05-18T20:47:31.273 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 8eeb80162074004e906b53d7340a12a14c471a83743aab975947e95ce061efcc @@ -394,7 +394,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c 30e2c43e4955db990e5b5a81e901f8aa74cc8820 F src/os_win.c 2a6c73eef01c51a048cc4ddccd57f981afbec18a F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c d82c877615ead80deccac6c7c59f7a875d7e3d7f1ff5652bbc74503d0e05f6d0 +F src/pager.c a26fa53c831f1f679352a0f1038933443f5ea36428bebb5e37e651fd98ad46a1 F src/pager.h 5e7b4e5afdcbdf558c211f27786672b3d2536003d0fb6c4888addb500c826e15 F src/parse.y 21660e5224d1e1635a4ad45ad4365c8f67153b8081b7a11e35629844ecb48ab0 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 @@ -609,7 +609,7 @@ F test/collateB.test 1e68906951b846570f29f20102ed91d29e634854ee47454d725f2151eca F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b F test/concfault.test 500f17c3fcfe7705114422bcc6ddd3c740001a43 -F test/concurrent.test 634b6a88f1942f5d68cc89d4d5efa2b11ba7913c +F test/concurrent.test 3eb5e6a911dc6ff72e3a679f563e683b436f6c701e6e1d6050173df2b8448d6b F test/concurrent2.test 77d655c6af93e77803b5c926555a838bb21f922f F test/concurrent3.test 0a5f7e3036d1eccf0782d7153ac21f5f222e9468 F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db @@ -1589,7 +1589,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 785c37d9dbdba9f3deedecbdd72f6761428ebfb58e9b9393b9ca4feab3f66f02 bb0d9281588b8cc24bf2f1f10d0c56277004226adaa2ce5037782503b283b45d -R 2209c7d58b85878752589360dcc54b3d -U drh -Z c0810d00013569d18eac37214e4c8809 +P 14ea84003600ada6f4605e47028cb72fe621a14bcce3637ad48b1512dcce41cd +R e81b507f65d4df5c681c42ea29f80e7d +U dan +Z 6bc66160d6824ab3a8c6e57fa6120412 diff --git a/manifest.uuid b/manifest.uuid index 6e8c4b9c4b..72434a6de3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -14ea84003600ada6f4605e47028cb72fe621a14bcce3637ad48b1512dcce41cd \ No newline at end of file +0eed152162b4721f7aaba8b480426476978a869e9578f100fca7b1d32942fe1a \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 253ca6ad4d..0d26c8d652 100644 --- a/src/pager.c +++ b/src/pager.c @@ -1795,6 +1795,7 @@ int sqlite3PagerBeginConcurrent(Pager *pPager){ int rc = SQLITE_OK; if( pPager->pAllRead==0 ){ pPager->pAllRead = sqlite3BitvecCreate(pPager->dbSize); + pPager->dbOrigSize = pPager->dbSize; if( pPager->pAllRead==0 ){ rc = SQLITE_NOMEM; } diff --git a/test/concurrent.test b/test/concurrent.test index 4e62d9e33f..92a9879ba8 100644 --- a/test/concurrent.test +++ b/test/concurrent.test @@ -526,5 +526,35 @@ do_multiclient_test tn { } {} } +do_multiclient_test tn { + do_test 6.$tn.1 { + sql1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t2 VALUES(2, 'two'); + } + } {wal} + + do_test 6.$tn.2 { + sql2 { + BEGIN CONCURRENT; + SELECT * FROM t2; + INSERT INTO t1 VALUES(3, 'three'); + } + } {2 two} + + do_test 6.$tn.3 { + sql1 { + INSERT INTO t2 VALUES(3, 'three'); + } + } {} + + do_test 6.$tn.2 { + list [catch { sql2 { COMMIT } } msg] $msg + } {1 {database is locked}} +} + finish_test From 55b36d5c070d46171a7cdd38156c8050ec99ebca Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 19 May 2017 19:57:15 +0000 Subject: [PATCH 046/179] Invoke sqlite3_log() in response to irregularities surrounding the Pager.pAllRead bit-vector. FossilOrigin-Name: 9527089b7aa3695cd577f31b263b4777e9bd62dbbc1bd3af892c570e52e8c3a1 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/bitvec.c | 7 ++++++- src/pager.c | 17 +++++++++++++++++ 4 files changed, 31 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index cf8e9ad002..779e15b1f4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\son\sthis\sbranch\scausing\ssome\spage-level\sread-locks\sto\sbe\somitted. -D 2017-05-18T20:47:31.273 +C Invoke\ssqlite3_log()\sin\sresponse\sto\sirregularities\ssurrounding\sthe\nPager.pAllRead\sbit-vector. +D 2017-05-19T19:57:15.749 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 8eeb80162074004e906b53d7340a12a14c471a83743aab975947e95ce061efcc @@ -348,7 +348,7 @@ F src/analyze.c 0d0ccf7520a201d8747ea2f02c92c26e26f801bc161f714f27b9f7630dde0421 F src/attach.c 8c476f8bd5d2afe11d925f890d30e527e5b0ce43 F src/auth.c 79f96c6f33bf0e5da8d1c282cee5ebb1852bb8a6ccca3e485d7c459b035d9c3c F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b -F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 +F src/bitvec.c fc0edcc2000f7d6faea86adcf9c2b8516882eb5aa2fc821fd2fd264e2cefde95 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca F src/btree.c b8ffd067e78704a8bf81b83c60a23987a46dac9aca0c08c3959482dfae02d197 F src/btree.h 14e99cc2b666beb60322173c761d16b668ec2e07c18bbb74e8a49fe85946f8a0 @@ -394,7 +394,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c 30e2c43e4955db990e5b5a81e901f8aa74cc8820 F src/os_win.c 2a6c73eef01c51a048cc4ddccd57f981afbec18a F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c a26fa53c831f1f679352a0f1038933443f5ea36428bebb5e37e651fd98ad46a1 +F src/pager.c ba57b91a11790725f53238c6e3f38734a296219f2cab26e0df764b7fb43c952e F src/pager.h 5e7b4e5afdcbdf558c211f27786672b3d2536003d0fb6c4888addb500c826e15 F src/parse.y 21660e5224d1e1635a4ad45ad4365c8f67153b8081b7a11e35629844ecb48ab0 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 @@ -1589,7 +1589,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 14ea84003600ada6f4605e47028cb72fe621a14bcce3637ad48b1512dcce41cd -R e81b507f65d4df5c681c42ea29f80e7d +P 0eed152162b4721f7aaba8b480426476978a869e9578f100fca7b1d32942fe1a +R dfb5691b0d7dbb9b1f618a1ddfa439f1 U dan -Z 6bc66160d6824ab3a8c6e57fa6120412 +Z 2e92764e6404f57fc047755474f473c5 diff --git a/manifest.uuid b/manifest.uuid index 72434a6de3..e02de0443e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0eed152162b4721f7aaba8b480426476978a869e9578f100fca7b1d32942fe1a \ No newline at end of file +9527089b7aa3695cd577f31b263b4777e9bd62dbbc1bd3af892c570e52e8c3a1 \ No newline at end of file diff --git a/src/bitvec.c b/src/bitvec.c index bd4a09429b..ae9c7f978d 100644 --- a/src/bitvec.c +++ b/src/bitvec.c @@ -170,7 +170,12 @@ int sqlite3BitvecSet(Bitvec *p, u32 i){ u32 h; if( p==0 ) return SQLITE_OK; assert( i>0 ); - assert( i<=p->iSize ); + /* assert( i<=p->iSize ); */ + if( i>p->iSize ){ + sqlite3_log(SQLITE_ERROR, + "Bitvec: setting bit %d of bitvec size %d\n", (int)i, (int)p->iSize + ); + } i--; while((p->iSize > BITVEC_NBIT) && p->iDivisor) { u32 bin = i/p->iDivisor; diff --git a/src/pager.c b/src/pager.c index 0d26c8d652..8c17a001bd 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5386,6 +5386,13 @@ int sqlite3PagerSharedLock(Pager *pPager){ }else{ pPager->eState = PAGER_READER; pPager->hasHeldSharedLock = 1; +#ifndef SQLITE_OMIT_CONCURRENT + if( pPager->pAllRead ){ + sqlite3_log(SQLITE_ERROR, + "Bitvec: pAllRead already allocated in PagerSharedLock()" + ); + } +#endif } return rc; } @@ -5884,6 +5891,16 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ ** file as well as into the page cache. Which would be incorrect in ** WAL mode. */ +#ifndef SQLITE_OMIT_CONCURRENT + if( pPager->pAllRead + && pPager->dbSize!=sqlite3BitvecSize(pPager->pAllRead) + ){ + sqlite3_log(SQLITE_ERROR, + "Bitvec: pAllRead size is %d, dbOrigSize set to %d", + (int)sqlite3BitvecSize(pPager->pAllRead), pPager->dbSize + ); + } +#endif pPager->eState = PAGER_WRITER_LOCKED; pPager->dbHintSize = pPager->dbSize; pPager->dbFileSize = pPager->dbSize; From aa59505ae85ee929b2d80d13e54ea17300f64e57 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 23 May 2017 19:23:45 +0000 Subject: [PATCH 047/179] Add experimental new API sqlite3_wal_info(). FossilOrigin-Name: 5b9d498f6e9de6ee2ab86370c02c28a2a8b83d717b96d23b1fc52107677e45a2 --- manifest | 26 +++++++++++++------------- manifest.uuid | 2 +- src/main.c | 31 +++++++++++++++++++++++++++++++ src/pager.c | 5 +++++ src/pager.h | 2 ++ src/sqlite.h.in | 25 +++++++++++++++++++++++++ src/test1.c | 36 ++++++++++++++++++++++++++++++++++++ src/wal.c | 16 ++++++++++++++++ src/wal.h | 3 +++ test/concurrent2.test | 29 +++++++++++++++++++++++++++++ 10 files changed, 161 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 779e15b1f4..8d9ba32435 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Invoke\ssqlite3_log()\sin\sresponse\sto\sirregularities\ssurrounding\sthe\nPager.pAllRead\sbit-vector. -D 2017-05-19T19:57:15.749 +C Add\sexperimental\snew\sAPI\ssqlite3_wal_info(). +D 2017-05-23T19:23:45.315 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 8eeb80162074004e906b53d7340a12a14c471a83743aab975947e95ce061efcc @@ -372,7 +372,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c d4bb3a135948553d18cf992f76f7ed7b18aa0327f250607b5a6671e55d9947d5 F src/legacy.c e88ed13c2d531decde75d42c2e35623fb9ce3cb0 F src/loadext.c a72909474dadce771d3669bf84bf689424f6f87d471fee898589c3ef9b2acfd9 -F src/main.c 1e448d204045c1dcd604853639d9d8fe253aa9ec302a2f1ffd2e22752fd6b708 +F src/main.c 5559b8e5aa6a7222d3dff52ae377b48e4365a7a4e3ea21491fc6c508a1fb838a F src/malloc.c e20bb2b48abec52d3faf01cce12e8b4f95973755fafec98d45162dfdab111978 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -394,8 +394,8 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c 30e2c43e4955db990e5b5a81e901f8aa74cc8820 F src/os_win.c 2a6c73eef01c51a048cc4ddccd57f981afbec18a F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c ba57b91a11790725f53238c6e3f38734a296219f2cab26e0df764b7fb43c952e -F src/pager.h 5e7b4e5afdcbdf558c211f27786672b3d2536003d0fb6c4888addb500c826e15 +F src/pager.c babddba9a8aa02feae4544175d4eeaee65321c263d78cf3ff3e0670df41bc837 +F src/pager.h 1afb4b777b3a297d56dc6cafc4f17d47d92522e9d2cb106839ea0fa0d6d489c7 F src/parse.y 21660e5224d1e1635a4ad45ad4365c8f67153b8081b7a11e35629844ecb48ab0 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 @@ -409,7 +409,7 @@ F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c d74b1cde1d9ca6d08bec50b60a5be19440273646bc8ae16648d748c38161d5b7 F src/shell.c a37d96b20b3644d0eb905df5aa7a0fcf9f6e73c15898337230c760a24a8df794 -F src/sqlite.h.in 8dd468837a4f6d76713e3a4cc65bea48095009038593d41040ab46c1b351197f +F src/sqlite.h.in 6eae6c87571d142f5aee27d49b13b5aec06bae40c706494e8565a64ab68b03f5 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 58fd0676d3111d02e62e5a35992a7d3da5d3f88753acc174f2d37b774fbbdd28 F src/sqliteInt.h b7e590bc90ebee66cf79fb41312a7cdbce881b7db316d934c159294ff6087fd5 @@ -417,7 +417,7 @@ F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 F src/tclsqlite.c c8cf60d0c5411d5e70e7c136470d29dbe760d250f55198b71682c67086524e4a -F src/test1.c c99f0442918a7a5d5b68a95d6024c211989e6c782c15ced5a558994baaf76a5e +F src/test1.c 108f95744b7c949ac806ad67a4be5d9e283bbe2011639a1b7ca512a1c52e908a F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5 F src/test3.c d03f5b5da9a2410b7a91c64b0d3306ed28ab6fee F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6 @@ -485,8 +485,8 @@ F src/vdbesort.c e72fe02a2121386ba767ede8942e9450878b8fc873abf3d1b6824485f092570 F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834 F src/vtab.c 35b9bdc2b41de32a417141d12097bcc4e29a77ed7cdb8f836d1d2305d946b61b F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 2858e71b30d521c80949fa10ed6b116367865594 -F src/wal.h 8659519a248ef0f33e575b20ab04211cebe1f430 +F src/wal.c 0c4faf368f51abb3f077fa5c175920b73b6a07584688a287c8d77296b6af582e +F src/wal.h 79378c5cd2127f32060d4952d827a7abdd490f8f9e5cbb14a4c39657ed99ad15 F src/walker.c b71a992b413b3a022572eccf29ef4b4890223791 F src/where.c c6352f15be5031907c68bcbde96cad1a6da20e9f4051d10168a59235de9a8566 F src/whereInt.h 2a4b634d63ce488b46d4b0da8f2eaa8f9aeab202bc25ef76f007de5e3fba1f20 @@ -610,7 +610,7 @@ F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b F test/concfault.test 500f17c3fcfe7705114422bcc6ddd3c740001a43 F test/concurrent.test 3eb5e6a911dc6ff72e3a679f563e683b436f6c701e6e1d6050173df2b8448d6b -F test/concurrent2.test 77d655c6af93e77803b5c926555a838bb21f922f +F test/concurrent2.test ba7e166187289ee23089fff4f9527bc36f7cc324196cc8e84d31eae2769e678f F test/concurrent3.test 0a5f7e3036d1eccf0782d7153ac21f5f222e9468 F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c @@ -1589,7 +1589,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0eed152162b4721f7aaba8b480426476978a869e9578f100fca7b1d32942fe1a -R dfb5691b0d7dbb9b1f618a1ddfa439f1 +P 9527089b7aa3695cd577f31b263b4777e9bd62dbbc1bd3af892c570e52e8c3a1 +R 37ee232af322d70fdc5a9c9d4ab6870b U dan -Z 2e92764e6404f57fc047755474f473c5 +Z 786d45f8ca40d966271275a2c1f1d5ca diff --git a/manifest.uuid b/manifest.uuid index e02de0443e..eb3f0245f3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9527089b7aa3695cd577f31b263b4777e9bd62dbbc1bd3af892c570e52e8c3a1 \ No newline at end of file +5b9d498f6e9de6ee2ab86370c02c28a2a8b83d717b96d23b1fc52107677e45a2 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 32d00087f5..57172a4a2e 100644 --- a/src/main.c +++ b/src/main.c @@ -4095,3 +4095,34 @@ void sqlite3_snapshot_free(sqlite3_snapshot *pSnapshot){ sqlite3_free(pSnapshot); } #endif /* SQLITE_ENABLE_SNAPSHOT */ + +SQLITE_EXPERIMENTAL int sqlite3_wal_info( + sqlite3 *db, const char *zDb, + unsigned int *pnPrior, unsigned int *pnFrame +){ + int rc = SQLITE_OK; + +#ifndef SQLITE_OMIT_WAL + Btree *pBt; + int iDb; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif + + sqlite3_mutex_enter(db->mutex); + iDb = sqlite3FindDbName(db, zDb); + if( iDb<0 ){ + return SQLITE_ERROR; + } + pBt = db->aDb[iDb].pBt; + rc = sqlite3PagerWalInfo(sqlite3BtreePager(pBt), pnPrior, pnFrame); + sqlite3_mutex_leave(db->mutex); +#endif /* SQLITE_OMIT_WAL */ + + return rc; +} + + diff --git a/src/pager.c b/src/pager.c index 8c17a001bd..4b0d5bcba6 100644 --- a/src/pager.c +++ b/src/pager.c @@ -7707,6 +7707,11 @@ int sqlite3PagerSnapshotRecover(Pager *pPager){ return rc; } #endif /* SQLITE_ENABLE_SNAPSHOT */ + +int sqlite3PagerWalInfo(Pager *pPager, u32 *pnPrior, u32 *pnFrame){ + return sqlite3WalInfo(pPager->pWal, pnPrior, pnFrame); +} + #endif /* !SQLITE_OMIT_WAL */ #ifdef SQLITE_ENABLE_ZIPVFS diff --git a/src/pager.h b/src/pager.h index 8bf2caa6f5..aa98bce95d 100644 --- a/src/pager.h +++ b/src/pager.h @@ -231,6 +231,8 @@ int sqlite3PagerIsWal(Pager*); int sqlite3PagerIswriteable(DbPage*); +int sqlite3PagerWalInfo(Pager*, u32 *pnPrior, u32 *pnFrame); + #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) void *sqlite3PagerCodec(DbPage *); #endif diff --git a/src/sqlite.h.in b/src/sqlite.h.in index d836e97058..df228b2c9a 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -8495,6 +8495,31 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( */ SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); +/* +** CAPI3REF: Wal related information regarding the most recent COMMIT +** EXPERIMENTAL +** +** This function reports on the state of the wal file (if any) for database +** zDb, which should be "main", "temp", or the name of the attached database. +** Its results - the values written to the output parameters - are only +** defined if the most recent SQL command on the connection was a successful +** COMMIT that wrote data to wal-mode database zDb. +** +** Assuming the above conditions are met, output parameter (*pnFrame) is set +** to the total number of frames in the wal file. Parameter (*pnPrior) is +** set to the number of frames that were present in the wal file before the +** most recent transaction was committed. So that the number of frames written +** by the most recent transaction is (*pnFrame)-(*pnPrior). +** +** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. It +** is not an error if this function is called at a time when the results +** are undefined. +*/ +SQLITE_EXPERIMENTAL int sqlite3_wal_info( + sqlite3 *db, const char *zDb, + unsigned int *pnPrior, unsigned int *pnFrame +); + /* ** Undo the hack that converts floating point types to integer for ** builds on processors without floating point support. diff --git a/src/test1.c b/src/test1.c index 87b255c9e9..f648f0559a 100644 --- a/src/test1.c +++ b/src/test1.c @@ -7369,6 +7369,41 @@ static int SQLITE_TCLAPI test_dbconfig_maindbname_icecube( } } +/* +** Usage: sqlite3_wal_info DB DBNAME +*/ +static int SQLITE_TCLAPI test_wal_info( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int rc; + sqlite3 *db; + char *zName; + unsigned int nPrior; + unsigned int nFrame; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + zName = Tcl_GetString(objv[2]); + + rc = sqlite3_wal_info(db, zName, &nPrior, &nFrame); + if( rc!=SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + return TCL_ERROR; + }else{ + Tcl_Obj *pNew = Tcl_NewObj(); + Tcl_ListObjAppendElement(interp, pNew, Tcl_NewWideIntObj((i64)nPrior)); + Tcl_ListObjAppendElement(interp, pNew, Tcl_NewWideIntObj((i64)nFrame)); + Tcl_SetObjResult(interp, pNew); + } + return TCL_OK; +} + /* ** Register commands with the TCL interpreter. */ @@ -7639,6 +7674,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite3_snapshot_cmp_blob", test_snapshot_cmp_blob, 0 }, #endif { "sqlite3_delete_database", test_delete_database, 0 }, + { "sqlite3_wal_info", test_wal_info, 0 }, }; static int bitmask_size = sizeof(Bitmask)*8; static int longdouble_size = sizeof(LONGDOUBLE_TYPE); diff --git a/src/wal.c b/src/wal.c index c56b1365cb..73b5f7ad82 100644 --- a/src/wal.c +++ b/src/wal.c @@ -446,6 +446,7 @@ struct Wal { WalIndexHdr hdr; /* Wal-index header for current transaction */ u32 minFrame; /* Ignore wal frames before this one */ u32 iReCksum; /* On commit, recalculate checksums from here */ + u32 nPriorFrame; /* For sqlite3WalInfo() */ const char *zWalName; /* Name of WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ #ifdef SQLITE_DEBUG @@ -2507,6 +2508,7 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ testcase( rc==SQLITE_PROTOCOL ); testcase( rc==SQLITE_OK ); + pWal->nPriorFrame = pWal->hdr.mxFrame; #ifdef SQLITE_ENABLE_SNAPSHOT if( rc==SQLITE_OK ){ if( pSnapshot && memcmp(pSnapshot, &pWal->hdr, sizeof(WalIndexHdr))!=0 ){ @@ -2948,6 +2950,7 @@ int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPage1, Bitvec *pAllRead){ } } + pWal->nPriorFrame = pWal->hdr.mxFrame; return rc; } @@ -3118,6 +3121,7 @@ static int walRestartLog(Wal *pWal){ ** to handle if this transaction is rolled back. */ walRestartHdr(pWal, salt1); walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); + pWal->nPriorFrame = 0; }else if( rc!=SQLITE_BUSY ){ return rc; } @@ -3759,4 +3763,16 @@ sqlite3_file *sqlite3WalFile(Wal *pWal){ return pWal->pWalFd; } +/* +** Return the values required by sqlite3_wal_info(). +*/ +int sqlite3WalInfo(Wal *pWal, u32 *pnPrior, u32 *pnFrame){ + int rc = SQLITE_OK; + if( pWal ){ + *pnFrame = pWal->hdr.mxFrame; + *pnPrior = pWal->nPriorFrame; + } + return rc; +} + #endif /* #ifndef SQLITE_OMIT_WAL */ diff --git a/src/wal.h b/src/wal.h index 5243bca162..b636d57a0a 100644 --- a/src/wal.h +++ b/src/wal.h @@ -153,5 +153,8 @@ int sqlite3WalFramesize(Wal *pWal); /* Return the sqlite3_file object for the WAL file */ sqlite3_file *sqlite3WalFile(Wal *pWal); +/* sqlite3_wal_info() data */ +int sqlite3WalInfo(Wal *pWal, u32 *pnPrior, u32 *pnFrame); + #endif /* ifndef SQLITE_OMIT_WAL */ #endif /* SQLITE_WAL_H */ diff --git a/test/concurrent2.test b/test/concurrent2.test index e0600ff9c3..6a9a40d16a 100644 --- a/test/concurrent2.test +++ b/test/concurrent2.test @@ -548,5 +548,34 @@ do_multiclient_test tn { } {} } +do_multiclient_test tn { + do_test 11.$tn.1 { + sql1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(a); + } + } {wal} + + do_test 11.$tn.2 { + code1 { sqlite3_wal_info db main } + } {0 2} + + do_test 11.$tn.3 { + sql1 { INSERT INTO t1 VALUES(1) } + code1 { sqlite3_wal_info db main } + } {2 3} + + do_test 11.$tn.4 { + sql2 { INSERT INTO t1 VALUES(2) } + code2 { sqlite3_wal_info db2 main } + } {3 4} + + do_test 11.$tn.5 { + sql1 { PRAGMA wal_checkpoint } + sql2 { INSERT INTO t1 VALUES(3) } + code2 { sqlite3_wal_info db2 main } + } {0 1} +} + finish_test From e3c3be85111b5a9e1b3891e3b3baf18576629195 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 25 May 2017 21:02:00 +0000 Subject: [PATCH 048/179] Fix a problem with the deferred page allocation on this branch that could occur when the database file is just slightly smaller than the PENDING_BYTE page offset. FossilOrigin-Name: 47a7dd92355ffd74645cf8da8186d5799c05a95907877a09c28de41135deeede --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/bitvec.c | 2 +- src/btree.c | 3 ++- test/concurrent2.test | 42 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 54 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 8d9ba32435..ad41d84290 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sexperimental\snew\sAPI\ssqlite3_wal_info(). -D 2017-05-23T19:23:45.315 +C Fix\sa\sproblem\swith\sthe\sdeferred\spage\sallocation\son\sthis\sbranch\sthat\scould\noccur\swhen\sthe\sdatabase\sfile\sis\sjust\sslightly\ssmaller\sthan\sthe\sPENDING_BYTE\npage\soffset. +D 2017-05-25T21:02:00.680 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 8eeb80162074004e906b53d7340a12a14c471a83743aab975947e95ce061efcc @@ -348,9 +348,9 @@ F src/analyze.c 0d0ccf7520a201d8747ea2f02c92c26e26f801bc161f714f27b9f7630dde0421 F src/attach.c 8c476f8bd5d2afe11d925f890d30e527e5b0ce43 F src/auth.c 79f96c6f33bf0e5da8d1c282cee5ebb1852bb8a6ccca3e485d7c459b035d9c3c F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b -F src/bitvec.c fc0edcc2000f7d6faea86adcf9c2b8516882eb5aa2fc821fd2fd264e2cefde95 +F src/bitvec.c e65c3d8ce932b4a38795574b549db6607b6140a22b82e783fa1ab57d0e42a49c F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c b8ffd067e78704a8bf81b83c60a23987a46dac9aca0c08c3959482dfae02d197 +F src/btree.c ffb001f02b4c65984068e50cbb308bb833c3d4d6280bf606e43d3eebf8d740df F src/btree.h 14e99cc2b666beb60322173c761d16b668ec2e07c18bbb74e8a49fe85946f8a0 F src/btreeInt.h 42c3e3d9534aed0b99ee68678b0311c33134c7c015037a319900eddd148584d6 F src/build.c ba3f389668754c407805bbc5f8ab140f063ba6b04a6a86f63006b63b3c7319a8 @@ -610,7 +610,7 @@ F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b F test/concfault.test 500f17c3fcfe7705114422bcc6ddd3c740001a43 F test/concurrent.test 3eb5e6a911dc6ff72e3a679f563e683b436f6c701e6e1d6050173df2b8448d6b -F test/concurrent2.test ba7e166187289ee23089fff4f9527bc36f7cc324196cc8e84d31eae2769e678f +F test/concurrent2.test 9dfbeb0a323733fe1d13443371734bb94a674dbf777f464365475903873111f8 F test/concurrent3.test 0a5f7e3036d1eccf0782d7153ac21f5f222e9468 F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c @@ -1589,7 +1589,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9527089b7aa3695cd577f31b263b4777e9bd62dbbc1bd3af892c570e52e8c3a1 -R 37ee232af322d70fdc5a9c9d4ab6870b +P 5b9d498f6e9de6ee2ab86370c02c28a2a8b83d717b96d23b1fc52107677e45a2 +R 707b1529cc7d8b55115b4608864612c4 U dan -Z 786d45f8ca40d966271275a2c1f1d5ca +Z fe29adbc1160b9b71ae2cc3a046207ff diff --git a/manifest.uuid b/manifest.uuid index eb3f0245f3..119515c7c8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5b9d498f6e9de6ee2ab86370c02c28a2a8b83d717b96d23b1fc52107677e45a2 \ No newline at end of file +47a7dd92355ffd74645cf8da8186d5799c05a95907877a09c28de41135deeede \ No newline at end of file diff --git a/src/bitvec.c b/src/bitvec.c index ae9c7f978d..a3e67b8f4c 100644 --- a/src/bitvec.c +++ b/src/bitvec.c @@ -170,7 +170,7 @@ int sqlite3BitvecSet(Bitvec *p, u32 i){ u32 h; if( p==0 ) return SQLITE_OK; assert( i>0 ); - /* assert( i<=p->iSize ); */ + assert( i<=p->iSize ); if( i>p->iSize ){ sqlite3_log(SQLITE_ERROR, "Bitvec: setting bit %d of bitvec size %d\n", (int)i, (int)p->iSize diff --git a/src/btree.c b/src/btree.c index b289b3f6c6..fa779386ae 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4129,10 +4129,11 @@ static int btreeFixUnlocked(Btree *p){ assert( nCurrent!=PENDING_BYTE_PAGE(pBt) ); sqlite3PagerSetDbsize(pBt->pPager, nCurrent); nFree = get4byte(&p1[36]); - nFin = MAX(nCurrent-nFree, nHPage); + nFin = nCurrent-nFree; if( nCurrent>PENDING_BYTE_PAGE(pBt) && nFin<=PENDING_BYTE_PAGE(pBt) ){ nFin--; } + nFin = MAX(nFin, nHPage); rc = btreeRelocateRange(pBt, nFin+1, nCurrent, 0); } diff --git a/test/concurrent2.test b/test/concurrent2.test index 6a9a40d16a..65f0022ab2 100644 --- a/test/concurrent2.test +++ b/test/concurrent2.test @@ -577,5 +577,47 @@ do_multiclient_test tn { } {0 1} } +reset_db +do_execsql_test 12.0 { + PRAGMA journal_mode = wal; + CREATE TABLE tx(a INTEGER PRIMARY KEY, b); +} {wal} +do_test 12.1 { + for {set i 0} {$i < 50} {incr i} { + execsql { + BEGIN CONCURRENT; + INSERT INTO tx(b) VALUES( randomblob( 1200 ) ); + COMMIT; + } + } + execsql { PRAGMA page_size } +} {1024} +do_execsql_test 12.2 { + DELETE FROM tx; +} +do_test 12.3 { + for {set i 0} {$i < 50} {incr i} { + execsql { + BEGIN CONCURRENT; + INSERT INTO tx(b) VALUES( randomblob( 1200 ) ); + COMMIT; + } + } + execsql { PRAGMA page_size } +} {1024} +do_execsql_test 12.4 { + DELETE FROM tx; +} +do_test 12.5 { + execsql { BEGIN CONCURRENT } + for {set i 0} {$i < 5000} {incr i} { + execsql { + INSERT INTO tx(b) VALUES( randomblob( 1200 ) ); + } + } + execsql { COMMIT } + execsql { PRAGMA page_size } +} {1024} + finish_test From 606f7184801e7e913d712e9a4645b326f3264b5e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 26 May 2017 16:15:05 +0000 Subject: [PATCH 049/179] Fix a problem with deferred page allocation in transactions that revert page allocations by savepoint rollbacks. FossilOrigin-Name: a4a3bbe64690856403642352f36e664a5a7fba686463a63446c6ada99df4e89f --- manifest | 13 ++++++------ manifest.uuid | 2 +- src/btree.c | 3 ++- test/concurrent4.test | 48 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+), 8 deletions(-) create mode 100644 test/concurrent4.test diff --git a/manifest b/manifest index ad41d84290..bb07f60ea8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sthe\sdeferred\spage\sallocation\son\sthis\sbranch\sthat\scould\noccur\swhen\sthe\sdatabase\sfile\sis\sjust\sslightly\ssmaller\sthan\sthe\sPENDING_BYTE\npage\soffset. -D 2017-05-25T21:02:00.680 +C Fix\sa\sproblem\swith\sdeferred\spage\sallocation\sin\stransactions\sthat\srevert\spage\nallocations\sby\ssavepoint\srollbacks. +D 2017-05-26T16:15:05.191 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 8eeb80162074004e906b53d7340a12a14c471a83743aab975947e95ce061efcc @@ -350,7 +350,7 @@ F src/auth.c 79f96c6f33bf0e5da8d1c282cee5ebb1852bb8a6ccca3e485d7c459b035d9c3c F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c e65c3d8ce932b4a38795574b549db6607b6140a22b82e783fa1ab57d0e42a49c F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c ffb001f02b4c65984068e50cbb308bb833c3d4d6280bf606e43d3eebf8d740df +F src/btree.c ca942fd75c1b791bd3fdf647e283473acf63e97d2252077216cdc1d56cb3a25e F src/btree.h 14e99cc2b666beb60322173c761d16b668ec2e07c18bbb74e8a49fe85946f8a0 F src/btreeInt.h 42c3e3d9534aed0b99ee68678b0311c33134c7c015037a319900eddd148584d6 F src/build.c ba3f389668754c407805bbc5f8ab140f063ba6b04a6a86f63006b63b3c7319a8 @@ -612,6 +612,7 @@ F test/concfault.test 500f17c3fcfe7705114422bcc6ddd3c740001a43 F test/concurrent.test 3eb5e6a911dc6ff72e3a679f563e683b436f6c701e6e1d6050173df2b8448d6b F test/concurrent2.test 9dfbeb0a323733fe1d13443371734bb94a674dbf777f464365475903873111f8 F test/concurrent3.test 0a5f7e3036d1eccf0782d7153ac21f5f222e9468 +F test/concurrent4.test d1b808b0c65f8a6368da1ca30bab9e25b0851f9dc529ffab761ba8c9a42d2943 F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c F test/conflict3.test a83db76a6c3503b2fa057c7bfb08c318d8a422202d8bc5b86226e078e5b49ff9 @@ -1589,7 +1590,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5b9d498f6e9de6ee2ab86370c02c28a2a8b83d717b96d23b1fc52107677e45a2 -R 707b1529cc7d8b55115b4608864612c4 +P 47a7dd92355ffd74645cf8da8186d5799c05a95907877a09c28de41135deeede +R 09d4e4c80f127f9ecf740a4309c9fec3 U dan -Z fe29adbc1160b9b71ae2cc3a046207ff +Z bd7d66ea928950ff2ef1b7b7ec07e968 diff --git a/manifest.uuid b/manifest.uuid index 119515c7c8..cea7e0cc0d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -47a7dd92355ffd74645cf8da8186d5799c05a95907877a09c28de41135deeede \ No newline at end of file +a4a3bbe64690856403642352f36e664a5a7fba686463a63446c6ada99df4e89f \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index fa779386ae..af57d4148e 100644 --- a/src/btree.c +++ b/src/btree.c @@ -522,6 +522,7 @@ static int btreePtrmapStore( pMap->aRollback[pMap->nRollback].pgno = pgno; pMap->aRollback[pMap->nRollback].parent = pMap->aPtr[iEntry].parent; pMap->aRollback[pMap->nRollback].eType = pMap->aPtr[iEntry].eType; + pMap->nRollback++; } /* Update the aPtr[] array */ @@ -538,7 +539,7 @@ static int btreePtrmapStore( */ static int btreePtrmapBegin(BtShared *pBt, int nSvpt){ BtreePtrmap *pMap = pBt->pMap; - if( pMap && nSvptnSvpt ){ + if( pMap && nSvpt>pMap->nSvpt ){ int i; if( nSvpt>=pMap->nSvptAlloc ){ int nNew = pMap->nSvptAlloc ? pMap->nSvptAlloc*2 : 16; diff --git a/test/concurrent4.test b/test/concurrent4.test new file mode 100644 index 0000000000..d129eaf7ac --- /dev/null +++ b/test/concurrent4.test @@ -0,0 +1,48 @@ +# 2017 May 26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Miscellaneous tests for transactions started with BEGIN CONCURRENT. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/wal_common.tcl +set ::testprefix concurrent4 + +ifcapable !concurrent { + finish_test + return +} + +do_execsql_test 1.0 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x PRIMARY KEY, y UNIQUE); + WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100) + INSERT INTO t1 SELECT randomblob(400), randomblob(400) FROM s; + DELETE FROM t1 WHERE rowid<2; +} {wal} + +do_execsql_test 1.1 { + BEGIN CONCURRENT; + INSERT INTO t1(rowid, x, y) VALUES(1000, randomblob(3000), randomblob(3000)); + SAVEPOINT abc; + DELETE FROM t1 WHERE rowid = 1000; +} + +do_execsql_test 1.2 { ROLLBACK TO abc } +do_execsql_test 1.3 { COMMIT } +do_execsql_test 1.4 { PRAGMA integrity_check } {ok} + + + +finish_test + From 7037787d303d6c26b74e6f62e317dc86d7381618 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 26 May 2017 16:51:15 +0000 Subject: [PATCH 050/179] Add extra test cases for deferred page allocation. FossilOrigin-Name: 9df0195780306a26709e87daabf9d426856aebf2082060b98818f2d48c7472ec --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/concurrent4.test | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index bb07f60ea8..cb5622a88f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sdeferred\spage\sallocation\sin\stransactions\sthat\srevert\spage\nallocations\sby\ssavepoint\srollbacks. -D 2017-05-26T16:15:05.191 +C Add\sextra\stest\scases\sfor\sdeferred\spage\sallocation. +D 2017-05-26T16:51:15.019 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 8eeb80162074004e906b53d7340a12a14c471a83743aab975947e95ce061efcc @@ -612,7 +612,7 @@ F test/concfault.test 500f17c3fcfe7705114422bcc6ddd3c740001a43 F test/concurrent.test 3eb5e6a911dc6ff72e3a679f563e683b436f6c701e6e1d6050173df2b8448d6b F test/concurrent2.test 9dfbeb0a323733fe1d13443371734bb94a674dbf777f464365475903873111f8 F test/concurrent3.test 0a5f7e3036d1eccf0782d7153ac21f5f222e9468 -F test/concurrent4.test d1b808b0c65f8a6368da1ca30bab9e25b0851f9dc529ffab761ba8c9a42d2943 +F test/concurrent4.test 989c6575225f9c4ef5d2392a9b9d0405665567c7501a3e44129598794d9b1b5f F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c F test/conflict3.test a83db76a6c3503b2fa057c7bfb08c318d8a422202d8bc5b86226e078e5b49ff9 @@ -1590,7 +1590,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 47a7dd92355ffd74645cf8da8186d5799c05a95907877a09c28de41135deeede -R 09d4e4c80f127f9ecf740a4309c9fec3 +P a4a3bbe64690856403642352f36e664a5a7fba686463a63446c6ada99df4e89f +R 69e13a62db6a845ec0ea0e91eabb688e U dan -Z bd7d66ea928950ff2ef1b7b7ec07e968 +Z 97d78c704135d26c80fab0abc59ae894 diff --git a/manifest.uuid b/manifest.uuid index cea7e0cc0d..c0fc82007e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a4a3bbe64690856403642352f36e664a5a7fba686463a63446c6ada99df4e89f \ No newline at end of file +9df0195780306a26709e87daabf9d426856aebf2082060b98818f2d48c7472ec \ No newline at end of file diff --git a/test/concurrent4.test b/test/concurrent4.test index d129eaf7ac..ce55cdd848 100644 --- a/test/concurrent4.test +++ b/test/concurrent4.test @@ -42,6 +42,47 @@ do_execsql_test 1.2 { ROLLBACK TO abc } do_execsql_test 1.3 { COMMIT } do_execsql_test 1.4 { PRAGMA integrity_check } {ok} +do_multiclient_test tn { + do_test 2.$tn.1 { + sql1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(a, b); + WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100) + INSERT INTO t1 SELECT randomblob(400), randomblob(400) FROM s; + + CREATE TABLE t2(a, b); + WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100) + INSERT INTO t2 SELECT randomblob(400), randomblob(400) FROM s; + } + + sql1 { + BEGIN CONCURRENT; + INSERT INTO t1 VALUES(randomblob(3000), randomblob(3000)); + } + } {} + + do_test 2.$tn.2 { + sql2 { + WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10) + INSERT INTO t2 SELECT randomblob(400), randomblob(400) FROM s; + } + sql2 { + DELETE FROM t2 WHERE rowid<10; + } + } {} + + do_test 2.$tn.3 { + sql1 { + COMMIT; + PRAGMA integrity_check; + } + } {ok} + do_test 2.$tn.4 { + sql2 { + PRAGMA integrity_check; + } + } {ok} +} finish_test From e3b047b3d7a666023cd29874e2676529cde0562e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 26 May 2017 18:18:51 +0000 Subject: [PATCH 051/179] Adjust the bitvec related sqlite3_log messages added by [9527089b]. FossilOrigin-Name: a7e0e7a4835314a04e065839a7f88b074eda795da2b2da5741b2afc096421a32 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/bitvec.c | 2 +- src/pager.c | 17 ----------------- 4 files changed, 9 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index cb5622a88f..61e52657d5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sextra\stest\scases\sfor\sdeferred\spage\sallocation. -D 2017-05-26T16:51:15.019 +C Adjust\sthe\sbitvec\srelated\ssqlite3_log\smessages\sadded\sby\s[9527089b]. +D 2017-05-26T18:18:51.758 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 8eeb80162074004e906b53d7340a12a14c471a83743aab975947e95ce061efcc @@ -348,7 +348,7 @@ F src/analyze.c 0d0ccf7520a201d8747ea2f02c92c26e26f801bc161f714f27b9f7630dde0421 F src/attach.c 8c476f8bd5d2afe11d925f890d30e527e5b0ce43 F src/auth.c 79f96c6f33bf0e5da8d1c282cee5ebb1852bb8a6ccca3e485d7c459b035d9c3c F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b -F src/bitvec.c e65c3d8ce932b4a38795574b549db6607b6140a22b82e783fa1ab57d0e42a49c +F src/bitvec.c c77b7f5759e413c1c8b53267d633c952e66db79c1171964c7e24c0f92f5019cf F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca F src/btree.c ca942fd75c1b791bd3fdf647e283473acf63e97d2252077216cdc1d56cb3a25e F src/btree.h 14e99cc2b666beb60322173c761d16b668ec2e07c18bbb74e8a49fe85946f8a0 @@ -394,7 +394,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c 30e2c43e4955db990e5b5a81e901f8aa74cc8820 F src/os_win.c 2a6c73eef01c51a048cc4ddccd57f981afbec18a F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c babddba9a8aa02feae4544175d4eeaee65321c263d78cf3ff3e0670df41bc837 +F src/pager.c 079884c8486596240e200f1e37e867b1ed56eff399e4bbbcabcc145dc5679cd5 F src/pager.h 1afb4b777b3a297d56dc6cafc4f17d47d92522e9d2cb106839ea0fa0d6d489c7 F src/parse.y 21660e5224d1e1635a4ad45ad4365c8f67153b8081b7a11e35629844ecb48ab0 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 @@ -1590,7 +1590,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a4a3bbe64690856403642352f36e664a5a7fba686463a63446c6ada99df4e89f -R 69e13a62db6a845ec0ea0e91eabb688e +P 9df0195780306a26709e87daabf9d426856aebf2082060b98818f2d48c7472ec +R ef448b68266e15a0b11a8140036a504f U dan -Z 97d78c704135d26c80fab0abc59ae894 +Z e753798d60e27266ad4ce7eb0092b295 diff --git a/manifest.uuid b/manifest.uuid index c0fc82007e..4b2ca7af23 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9df0195780306a26709e87daabf9d426856aebf2082060b98818f2d48c7472ec \ No newline at end of file +a7e0e7a4835314a04e065839a7f88b074eda795da2b2da5741b2afc096421a32 \ No newline at end of file diff --git a/src/bitvec.c b/src/bitvec.c index a3e67b8f4c..31994f550b 100644 --- a/src/bitvec.c +++ b/src/bitvec.c @@ -171,7 +171,7 @@ int sqlite3BitvecSet(Bitvec *p, u32 i){ if( p==0 ) return SQLITE_OK; assert( i>0 ); assert( i<=p->iSize ); - if( i>p->iSize ){ + if( i>p->iSize || i==0 ){ sqlite3_log(SQLITE_ERROR, "Bitvec: setting bit %d of bitvec size %d\n", (int)i, (int)p->iSize ); diff --git a/src/pager.c b/src/pager.c index 4b0d5bcba6..cbc078ffe6 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5386,13 +5386,6 @@ int sqlite3PagerSharedLock(Pager *pPager){ }else{ pPager->eState = PAGER_READER; pPager->hasHeldSharedLock = 1; -#ifndef SQLITE_OMIT_CONCURRENT - if( pPager->pAllRead ){ - sqlite3_log(SQLITE_ERROR, - "Bitvec: pAllRead already allocated in PagerSharedLock()" - ); - } -#endif } return rc; } @@ -5891,16 +5884,6 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ ** file as well as into the page cache. Which would be incorrect in ** WAL mode. */ -#ifndef SQLITE_OMIT_CONCURRENT - if( pPager->pAllRead - && pPager->dbSize!=sqlite3BitvecSize(pPager->pAllRead) - ){ - sqlite3_log(SQLITE_ERROR, - "Bitvec: pAllRead size is %d, dbOrigSize set to %d", - (int)sqlite3BitvecSize(pPager->pAllRead), pPager->dbSize - ); - } -#endif pPager->eState = PAGER_WRITER_LOCKED; pPager->dbHintSize = pPager->dbSize; pPager->dbFileSize = pPager->dbSize; From 7fff2e1cb9c838edc392ee1edd9070fac53d96bb Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 29 May 2017 14:27:37 +0000 Subject: [PATCH 052/179] Enhance the log messages emitted when a page conflict is detected. FossilOrigin-Name: 92618492b048867af38922825f3d094eaaa2dd919b1ed2f7372483cc53f892bf --- manifest | 17 +++---- manifest.uuid | 2 +- src/btree.c | 38 ++++++++++++--- src/btreeInt.h | 3 ++ src/wal.c | 26 ++++++++-- test/concurrent5.test | 109 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 174 insertions(+), 21 deletions(-) create mode 100644 test/concurrent5.test diff --git a/manifest b/manifest index 61e52657d5..9c926fb54c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adjust\sthe\sbitvec\srelated\ssqlite3_log\smessages\sadded\sby\s[9527089b]. -D 2017-05-26T18:18:51.758 +C Enhance\sthe\slog\smessages\semitted\swhen\sa\spage\sconflict\sis\sdetected. +D 2017-05-29T14:27:37.696 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 8eeb80162074004e906b53d7340a12a14c471a83743aab975947e95ce061efcc @@ -350,9 +350,9 @@ F src/auth.c 79f96c6f33bf0e5da8d1c282cee5ebb1852bb8a6ccca3e485d7c459b035d9c3c F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c c77b7f5759e413c1c8b53267d633c952e66db79c1171964c7e24c0f92f5019cf F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c ca942fd75c1b791bd3fdf647e283473acf63e97d2252077216cdc1d56cb3a25e +F src/btree.c 3c5409453a17294ed8fe7b44dc87c0434b2418f447ad768f4624ead69d5dfa18 F src/btree.h 14e99cc2b666beb60322173c761d16b668ec2e07c18bbb74e8a49fe85946f8a0 -F src/btreeInt.h 42c3e3d9534aed0b99ee68678b0311c33134c7c015037a319900eddd148584d6 +F src/btreeInt.h 7429915fc8f51bbd78b7ac023aa4afbe5b9660fc1e6970f144b07540a34a4623 F src/build.c ba3f389668754c407805bbc5f8ab140f063ba6b04a6a86f63006b63b3c7319a8 F src/callback.c 2e76147783386374bf01b227f752c81ec872d730 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e @@ -485,7 +485,7 @@ F src/vdbesort.c e72fe02a2121386ba767ede8942e9450878b8fc873abf3d1b6824485f092570 F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834 F src/vtab.c 35b9bdc2b41de32a417141d12097bcc4e29a77ed7cdb8f836d1d2305d946b61b F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 0c4faf368f51abb3f077fa5c175920b73b6a07584688a287c8d77296b6af582e +F src/wal.c 4b857e74548bb0f9bc0c7f56517327421f925b1263ac65a6e58e9c176194857e F src/wal.h 79378c5cd2127f32060d4952d827a7abdd490f8f9e5cbb14a4c39657ed99ad15 F src/walker.c b71a992b413b3a022572eccf29ef4b4890223791 F src/where.c c6352f15be5031907c68bcbde96cad1a6da20e9f4051d10168a59235de9a8566 @@ -613,6 +613,7 @@ F test/concurrent.test 3eb5e6a911dc6ff72e3a679f563e683b436f6c701e6e1d6050173df2b F test/concurrent2.test 9dfbeb0a323733fe1d13443371734bb94a674dbf777f464365475903873111f8 F test/concurrent3.test 0a5f7e3036d1eccf0782d7153ac21f5f222e9468 F test/concurrent4.test 989c6575225f9c4ef5d2392a9b9d0405665567c7501a3e44129598794d9b1b5f +F test/concurrent5.test 06fd0b5294586d170a3b12d68153394fbfe642c8346e249a6240bc6ababc5f4b F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c F test/conflict3.test a83db76a6c3503b2fa057c7bfb08c318d8a422202d8bc5b86226e078e5b49ff9 @@ -1590,7 +1591,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9df0195780306a26709e87daabf9d426856aebf2082060b98818f2d48c7472ec -R ef448b68266e15a0b11a8140036a504f +P a7e0e7a4835314a04e065839a7f88b074eda795da2b2da5741b2afc096421a32 +R beb58ff81a99fbcff0806ad7e4f3b8ee U dan -Z e753798d60e27266ad4ce7eb0092b295 +Z a8b4d17828fb801d4b09ca76008d7042 diff --git a/manifest.uuid b/manifest.uuid index 4b2ca7af23..d27dbec941 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a7e0e7a4835314a04e065839a7f88b074eda795da2b2da5741b2afc096421a32 \ No newline at end of file +92618492b048867af38922825f3d094eaaa2dd919b1ed2f7372483cc53f892bf \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index af57d4148e..e9a09b69bd 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2289,6 +2289,17 @@ static int getAndInitPage( return rc; } +#ifndef SQLITE_OMIT_CONCURRENT +/* +** Set the value of the MemPage.pgnoRoot variable, if it exists. +*/ +static void setMempageRoot(MemPage *pPg, u32 pgnoRoot){ + pPg->pgnoRoot = pgnoRoot; +} +#else +# define setMempageRoot(x,y) +#endif + /* ** Release a MemPage. This should be called once for each prior ** call to btreeGetPage. @@ -5222,6 +5233,7 @@ const void *sqlite3BtreePayloadFetch(BtCursor *pCur, u32 *pAmt){ */ static int moveToChild(BtCursor *pCur, u32 newPgno){ BtShared *pBt = pCur->pBt; + int rc; assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState==CURSOR_VALID ); @@ -5234,8 +5246,12 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){ pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); pCur->aiIdx[pCur->iPage++] = pCur->ix; pCur->ix = 0; - return getAndInitPage(pBt, newPgno, &pCur->apPage[pCur->iPage], + rc = getAndInitPage(pBt, newPgno, &pCur->apPage[pCur->iPage], pCur, pCur->curPagerFlags); + if( rc==SQLITE_OK ){ + setMempageRoot(pCur->apPage[pCur->iPage], pCur->pgnoRoot); + } + return rc; } #ifdef SQLITE_DEBUG @@ -5341,6 +5357,7 @@ static int moveToRoot(BtCursor *pCur){ pCur->eState = CURSOR_INVALID; return rc; } + setMempageRoot(pCur->apPage[0], pCur->pgnoRoot); pCur->iPage = 0; pCur->curIntKey = pCur->apPage[0]->intKey; } @@ -7500,7 +7517,8 @@ static int balance_nonroot( int iParentIdx, /* Index of "the page" in pParent */ u8 *aOvflSpace, /* page-size bytes of space for parent ovfl */ int isRoot, /* True if pParent is a root-page */ - int bBulk /* True if this call is part of a bulk load */ + int bBulk, /* True if this call is part of a bulk load */ + Pgno pgnoRoot /* Root page of b-tree being balanced */ ){ BtShared *pBt; /* The whole database */ int nMaxCells = 0; /* Allocated size of apCell, szCell, aFrom. */ @@ -7592,6 +7610,8 @@ static int balance_nonroot( memset(apOld, 0, (i+1)*sizeof(MemPage*)); goto balance_cleanup; } + setMempageRoot(apOld[i], pgnoRoot); + nMaxCells += 1+apOld[i]->nCell+apOld[i]->nOverflow; if( (i--)==0 ) break; @@ -8400,7 +8420,7 @@ static int balance(BtCursor *pCur){ */ u8 *pSpace = sqlite3PageMalloc(pCur->pBt->pageSize); rc = balance_nonroot(pParent, iIdx, pSpace, iPage==1, - pCur->hints&BTREE_BULKLOAD); + pCur->hints&BTREE_BULKLOAD, pCur->pgnoRoot); if( pFree ){ /* If pFree is not NULL, it points to the pSpace buffer used ** by a previous call to balance_nonroot(). Its contents are @@ -8998,7 +9018,8 @@ static int clearDatabasePage( BtShared *pBt, /* The BTree that contains the table */ Pgno pgno, /* Page number to clear */ int freePageFlag, /* Deallocate page if true */ - int *pnChange /* Add number of Cells freed to this counter */ + int *pnChange, /* Add number of Cells freed to this counter */ + Pgno pgnoRoot ){ MemPage *pPage; int rc; @@ -9013,6 +9034,7 @@ static int clearDatabasePage( } rc = getAndInitPage(pBt, pgno, &pPage, 0, 0); if( rc ) return rc; + setMempageRoot(pPage, pgnoRoot); if( pPage->bBusy ){ rc = SQLITE_CORRUPT_BKPT; goto cleardatabasepage_out; @@ -9022,14 +9044,16 @@ static int clearDatabasePage( for(i=0; inCell; i++){ pCell = findCell(pPage, i); if( !pPage->leaf ){ - rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange); + rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange, pgnoRoot); if( rc ) goto cleardatabasepage_out; } rc = clearCell(pPage, pCell, &info); if( rc ) goto cleardatabasepage_out; } if( !pPage->leaf ){ - rc = clearDatabasePage(pBt, get4byte(&pPage->aData[hdr+8]), 1, pnChange); + rc = clearDatabasePage( + pBt, get4byte(&pPage->aData[hdr+8]), 1, pnChange, pgnoRoot + ); if( rc ) goto cleardatabasepage_out; }else if( pnChange ){ assert( pPage->intKey || CORRUPT_DB ); @@ -9074,7 +9098,7 @@ int sqlite3BtreeClearTable(Btree *p, int iTable, int *pnChange){ ** is the root of a table b-tree - if it is not, the following call is ** a no-op). */ invalidateIncrblobCursors(p, (Pgno)iTable, 0, 1); - rc = clearDatabasePage(pBt, (Pgno)iTable, 0, pnChange); + rc = clearDatabasePage(pBt, (Pgno)iTable, 0, pnChange, (Pgno)iTable); } sqlite3BtreeLeave(p); return rc; diff --git a/src/btreeInt.h b/src/btreeInt.h index d924f08ff2..5820db64c3 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -301,6 +301,9 @@ struct MemPage { DbPage *pDbPage; /* Pager page handle */ u16 (*xCellSize)(MemPage*,u8*); /* cellSizePtr method */ void (*xParseCell)(MemPage*,u8*,CellInfo*); /* btreeParseCell method */ +#ifndef SQLITE_OMIT_CONCURRENT + u32 pgnoRoot; /* Root page of b-tree that this page belongs to */ +#endif }; /* diff --git a/src/wal.c b/src/wal.c index 73b5f7ad82..4139b7c563 100644 --- a/src/wal.c +++ b/src/wal.c @@ -243,6 +243,7 @@ #ifndef SQLITE_OMIT_WAL #include "wal.h" +#include "btreeInt.h" /* ** Trace output macros @@ -2916,11 +2917,26 @@ int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPage1, Bitvec *pAllRead){ rc = SQLITE_BUSY_SNAPSHOT; } }else if( sqlite3BitvecTestNotNull(pAllRead, aPgno[i]) ){ - sqlite3_log(SQLITE_OK, - "cannot commit CONCURRENT transaction (conflict at page %d)", - (int)aPgno[i] - ); - rc = SQLITE_BUSY_SNAPSHOT; + PgHdr *pPg = 0; + rc = sqlite3PagerGet(pPage1->pPager, aPgno[i], &pPg, 0); + if( rc==SQLITE_OK ){ + Pgno pgnoRoot = 0; + int bWrite = -1; + if( pPg ){ + pgnoRoot = ((MemPage*)sqlite3PagerGetExtra(pPg))->pgnoRoot; + bWrite = sqlite3PagerIswriteable(pPg); + sqlite3PagerUnref(pPg); + } + sqlite3_log(SQLITE_OK, + "cannot commit CONCURRENT transaction " + "- conflict at page %d " + "(%s page; part of b-tree with root page %d)", + (int)aPgno[i], + (bWrite==0?"read-only":(bWrite>0?"read/write":"unknown")), + (int)pgnoRoot + ); + rc = SQLITE_BUSY_SNAPSHOT; + } }else if( (pPg = sqlite3PagerLookup(pPager, aPgno[i])) ){ /* Page aPgno[i], which is present in the pager cache, has been ** modified since the current CONCURRENT transaction was started. diff --git a/test/concurrent5.test b/test/concurrent5.test new file mode 100644 index 0000000000..e77c794048 --- /dev/null +++ b/test/concurrent5.test @@ -0,0 +1,109 @@ +# 2017 May 26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/wal_common.tcl +set ::testprefix concurrent5 + +ifcapable !concurrent { + finish_test + return +} + +db close +sqlite3_shutdown +test_sqlite3_log [list lappend ::log] +set ::log [list] + +sqlite3 db test.db + +proc do_test_conflict_msg {tn msg} { + set msg "cannot commit CONCURRENT transaction - [string trim $msg]" + uplevel [list do_test $tn {lindex $::log end} $msg] +} + +do_execsql_test 1.0 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x, y); + CREATE TABLE t2(c); + WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100) + INSERT INTO t1 SELECT randomblob(400), randomblob(400) FROM s; +} {wal} + +sqlite3 db2 test.db + +do_test 1.1.1 { + set ::log [list] + db2 eval { + BEGIN CONCURRENT; + SELECT count(*) FROM t1; + INSERT INTO t2 VALUES(10); + } + + db eval { + INSERT INTO t1 VALUES(randomblob(200), randomblob(200)); + } + + catchsql COMMIT db2 +} {1 {database is locked}} +do_test_conflict_msg 1.1.2 { + conflict at page 2 (read-only page; part of b-tree with root page 2) +} + +do_test 1.2.1 { + set ::log [list] + db2 eval { + ROLLBACK; + BEGIN CONCURRENT; + INSERT INTO t1 VALUES(11, 12); + } + + db eval { + INSERT INTO t1 VALUES(12, 11); + } + + catchsql COMMIT db2 +} {1 {database is locked}} + +do_test_conflict_msg 1.2.2 { + conflict at page 105 (read/write page; part of b-tree with root page 2) +} + +do_test 1.3.1 { + set ::log [list] + db2 eval { + ROLLBACK; + BEGIN CONCURRENT; + INSERT INTO t2 VALUES('x'); + } + + db eval { + INSERT INTO t2 VALUES('y'); + } + + catchsql COMMIT db2 +} {1 {database is locked}} + +do_test_conflict_msg 1.3.2 { + conflict at page 3 (read/write page; part of b-tree with root page 3) +} + +db close +db2 close +sqlite3_shutdown +test_sqlite3_log +sqlite3_initialize +finish_test + From 995b2457e2666e18d89ccb523bf8cd47482b9622 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 29 May 2017 19:23:56 +0000 Subject: [PATCH 053/179] Instead of a root page number, log the object (table or index) name if a page level locking conflict is detected. FossilOrigin-Name: 9ad846e57bd427adc7c29768cabca18905f7f978168e0642a5917d894fda8bfd --- manifest | 22 +++++++++--------- manifest.uuid | 2 +- src/btree.c | 52 ++++++++++++++++++++++++++++++++++++++++++- src/pager.c | 8 ++++--- src/pager.h | 2 +- src/wal.c | 30 +++++++------------------ src/wal.h | 2 +- test/concurrent5.test | 30 ++++++++++++++++++++++--- 8 files changed, 105 insertions(+), 43 deletions(-) diff --git a/manifest b/manifest index 9c926fb54c..7d09c7ae3f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\slog\smessages\semitted\swhen\sa\spage\sconflict\sis\sdetected. -D 2017-05-29T14:27:37.696 +C Instead\sof\sa\sroot\spage\snumber,\slog\sthe\sobject\s(table\sor\sindex)\sname\sif\sa\spage\nlevel\slocking\sconflict\sis\sdetected. +D 2017-05-29T19:23:56.135 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 8eeb80162074004e906b53d7340a12a14c471a83743aab975947e95ce061efcc @@ -350,7 +350,7 @@ F src/auth.c 79f96c6f33bf0e5da8d1c282cee5ebb1852bb8a6ccca3e485d7c459b035d9c3c F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c c77b7f5759e413c1c8b53267d633c952e66db79c1171964c7e24c0f92f5019cf F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 3c5409453a17294ed8fe7b44dc87c0434b2418f447ad768f4624ead69d5dfa18 +F src/btree.c 8be55b3e9ed1346b5cfd3f749e6c338ced1cade9242a3c5404696bc4aed26652 F src/btree.h 14e99cc2b666beb60322173c761d16b668ec2e07c18bbb74e8a49fe85946f8a0 F src/btreeInt.h 7429915fc8f51bbd78b7ac023aa4afbe5b9660fc1e6970f144b07540a34a4623 F src/build.c ba3f389668754c407805bbc5f8ab140f063ba6b04a6a86f63006b63b3c7319a8 @@ -394,8 +394,8 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c 30e2c43e4955db990e5b5a81e901f8aa74cc8820 F src/os_win.c 2a6c73eef01c51a048cc4ddccd57f981afbec18a F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c 079884c8486596240e200f1e37e867b1ed56eff399e4bbbcabcc145dc5679cd5 -F src/pager.h 1afb4b777b3a297d56dc6cafc4f17d47d92522e9d2cb106839ea0fa0d6d489c7 +F src/pager.c 817afaac17f7c4b1282d9aa283f10fc4b58d5d1e4afb12f84c46038904311650 +F src/pager.h ba7b3fbb18f78835a7368d66530bf21d6782e3e640d79c9550d218742c0b07ce F src/parse.y 21660e5224d1e1635a4ad45ad4365c8f67153b8081b7a11e35629844ecb48ab0 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 @@ -485,8 +485,8 @@ F src/vdbesort.c e72fe02a2121386ba767ede8942e9450878b8fc873abf3d1b6824485f092570 F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834 F src/vtab.c 35b9bdc2b41de32a417141d12097bcc4e29a77ed7cdb8f836d1d2305d946b61b F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 4b857e74548bb0f9bc0c7f56517327421f925b1263ac65a6e58e9c176194857e -F src/wal.h 79378c5cd2127f32060d4952d827a7abdd490f8f9e5cbb14a4c39657ed99ad15 +F src/wal.c e006353c3578071a457888aedd5eec9c8f1ba31ba11ddac4084c91c80478ca16 +F src/wal.h 1ea51dc499d6451529b822a8aaac053eafeef10b7fd9e5a4c9cc413182be429f F src/walker.c b71a992b413b3a022572eccf29ef4b4890223791 F src/where.c c6352f15be5031907c68bcbde96cad1a6da20e9f4051d10168a59235de9a8566 F src/whereInt.h 2a4b634d63ce488b46d4b0da8f2eaa8f9aeab202bc25ef76f007de5e3fba1f20 @@ -613,7 +613,7 @@ F test/concurrent.test 3eb5e6a911dc6ff72e3a679f563e683b436f6c701e6e1d6050173df2b F test/concurrent2.test 9dfbeb0a323733fe1d13443371734bb94a674dbf777f464365475903873111f8 F test/concurrent3.test 0a5f7e3036d1eccf0782d7153ac21f5f222e9468 F test/concurrent4.test 989c6575225f9c4ef5d2392a9b9d0405665567c7501a3e44129598794d9b1b5f -F test/concurrent5.test 06fd0b5294586d170a3b12d68153394fbfe642c8346e249a6240bc6ababc5f4b +F test/concurrent5.test d5d7d9d404a9b4502464fc097c1fc5c3012bb4f1b063fae7ad707ca983fc86c5 F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c F test/conflict3.test a83db76a6c3503b2fa057c7bfb08c318d8a422202d8bc5b86226e078e5b49ff9 @@ -1591,7 +1591,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a7e0e7a4835314a04e065839a7f88b074eda795da2b2da5741b2afc096421a32 -R beb58ff81a99fbcff0806ad7e4f3b8ee +P 92618492b048867af38922825f3d094eaaa2dd919b1ed2f7372483cc53f892bf +R 9903c08299ff258c04c9dfdcdc90aac6 U dan -Z a8b4d17828fb801d4b09ca76008d7042 +Z 4393c020077681d651727b37beafcbae diff --git a/manifest.uuid b/manifest.uuid index d27dbec941..cee4fdfd92 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -92618492b048867af38922825f3d094eaaa2dd919b1ed2f7372483cc53f892bf \ No newline at end of file +9ad846e57bd427adc7c29768cabca18905f7f978168e0642a5917d894fda8bfd \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index e9a09b69bd..38dfbccf41 100644 --- a/src/btree.c +++ b/src/btree.c @@ -10286,10 +10286,60 @@ int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); } */ int sqlite3BtreeExclusiveLock(Btree *p){ int rc; + Pgno pgno = 0; BtShared *pBt = p->pBt; assert( p->inTrans==TRANS_WRITE && pBt->pPage1 ); sqlite3BtreeEnter(p); - rc = sqlite3PagerExclusiveLock(pBt->pPager, pBt->pPage1->pDbPage); + rc = sqlite3PagerExclusiveLock(pBt->pPager, pBt->pPage1->pDbPage, &pgno); + if( rc==SQLITE_BUSY_SNAPSHOT && pgno ){ + PgHdr *pPg = 0; + int rc2 = sqlite3PagerGet(pBt->pPager, pgno, &pPg, 0); + if( rc2==SQLITE_OK ){ + int bWrite = -1; + const char *zObj = 0; + const char *zTab = 0; + + if( pPg ){ + Pgno pgnoRoot = 0; + HashElem *pE; + Schema *pSchema; + + pgnoRoot = ((MemPage*)sqlite3PagerGetExtra(pPg))->pgnoRoot; + bWrite = sqlite3PagerIswriteable(pPg); + sqlite3PagerUnref(pPg); + + pSchema = sqlite3SchemaGet(p->db, p); + if( pSchema ){ + for(pE=sqliteHashFirst(&pSchema->tblHash); pE; pE=sqliteHashNext(pE)){ + Table *pTab = (Table *)sqliteHashData(pE); + if( pTab->tnum==(int)pgnoRoot ){ + zObj = pTab->zName; + zTab = 0; + }else{ + Index *pIdx; + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->tnum==(int)pgnoRoot ){ + zObj = pIdx->zName; + zTab = pTab->zName; + } + } + } + } + } + } + + sqlite3_log(SQLITE_OK, + "cannot commit CONCURRENT transaction " + "- conflict at page %d " + "(%s page; part of db %s %s%s%s)", + (int)pgno, + (bWrite==0?"read-only":(bWrite>0?"read/write":"unknown")), + (zTab ? "index" : "table"), + (zTab ? zTab : ""), (zTab ? "." : ""), (zObj ? zObj : "UNKNOWN") + ); + } + } + sqlite3BtreeLeave(p); return rc; } diff --git a/src/pager.c b/src/pager.c index cbc078ffe6..0ea9fc5d57 100644 --- a/src/pager.c +++ b/src/pager.c @@ -4266,7 +4266,7 @@ static int syncJournal(Pager *pPager, int newHdr){ assert( assert_pager_state(pPager) ); assert( !pagerUseWal(pPager) ); - rc = sqlite3PagerExclusiveLock(pPager, 0); + rc = sqlite3PagerExclusiveLock(pPager, 0, 0); if( rc!=SQLITE_OK ) return rc; if( !pPager->noSync ){ @@ -6347,7 +6347,7 @@ int sqlite3PagerSync(Pager *pPager, const char *zMaster){ ** is returned. Otherwise, if some other error occurs (IO error, OOM etc.), ** and SQLite error code is returned. */ -int sqlite3PagerExclusiveLock(Pager *pPager, PgHdr *pPage1){ +int sqlite3PagerExclusiveLock(Pager *pPager, PgHdr *pPage1, Pgno *piConflict){ int rc = pPager->errCode; assert( assert_pager_state(pPager) ); if( rc==SQLITE_OK ){ @@ -6367,7 +6367,9 @@ int sqlite3PagerExclusiveLock(Pager *pPager, PgHdr *pPage1){ ** invoke the busy-handler and try again for as long as it returns ** non-zero. */ do { - rc = sqlite3WalLockForCommit(pPager->pWal, pPage1, pPager->pAllRead); + rc = sqlite3WalLockForCommit( + pPager->pWal, pPage1, pPager->pAllRead, piConflict + ); }while( rc==SQLITE_BUSY && pPager->xBusyHandler(pPager->pBusyHandlerArg) ); diff --git a/src/pager.h b/src/pager.h index aa98bce95d..8aa17c4553 100644 --- a/src/pager.h +++ b/src/pager.h @@ -164,7 +164,7 @@ void *sqlite3PagerGetExtra(DbPage *); void sqlite3PagerPagecount(Pager*, int*); int sqlite3PagerBegin(Pager*, int exFlag, int); int sqlite3PagerCommitPhaseOne(Pager*,const char *zMaster, int); -int sqlite3PagerExclusiveLock(Pager*, DbPage *pPage1); +int sqlite3PagerExclusiveLock(Pager*, DbPage *pPage1, Pgno*); int sqlite3PagerSync(Pager *pPager, const char *zMaster); int sqlite3PagerCommitPhaseTwo(Pager*); int sqlite3PagerRollback(Pager*); diff --git a/src/wal.c b/src/wal.c index 4139b7c563..13e1c57a99 100644 --- a/src/wal.c +++ b/src/wal.c @@ -243,7 +243,6 @@ #ifndef SQLITE_OMIT_WAL #include "wal.h" -#include "btreeInt.h" /* ** Trace output macros @@ -2857,7 +2856,12 @@ static int walUpgradeReadlock(Wal *pWal){ ** if an error (i.e. an OOM condition or IO error), an SQLite error code ** is returned. */ -int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPage1, Bitvec *pAllRead){ +int sqlite3WalLockForCommit( + Wal *pWal, + PgHdr *pPage1, + Bitvec *pAllRead, + Pgno *piConflict +){ Pager *pPager = pPage1->pPager; int rc = walWriteLock(pWal); @@ -2917,26 +2921,8 @@ int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPage1, Bitvec *pAllRead){ rc = SQLITE_BUSY_SNAPSHOT; } }else if( sqlite3BitvecTestNotNull(pAllRead, aPgno[i]) ){ - PgHdr *pPg = 0; - rc = sqlite3PagerGet(pPage1->pPager, aPgno[i], &pPg, 0); - if( rc==SQLITE_OK ){ - Pgno pgnoRoot = 0; - int bWrite = -1; - if( pPg ){ - pgnoRoot = ((MemPage*)sqlite3PagerGetExtra(pPg))->pgnoRoot; - bWrite = sqlite3PagerIswriteable(pPg); - sqlite3PagerUnref(pPg); - } - sqlite3_log(SQLITE_OK, - "cannot commit CONCURRENT transaction " - "- conflict at page %d " - "(%s page; part of b-tree with root page %d)", - (int)aPgno[i], - (bWrite==0?"read-only":(bWrite>0?"read/write":"unknown")), - (int)pgnoRoot - ); - rc = SQLITE_BUSY_SNAPSHOT; - } + *piConflict = aPgno[i]; + rc = SQLITE_BUSY_SNAPSHOT; }else if( (pPg = sqlite3PagerLookup(pPager, aPgno[i])) ){ /* Page aPgno[i], which is present in the pager cache, has been ** modified since the current CONCURRENT transaction was started. diff --git a/src/wal.h b/src/wal.h index b636d57a0a..d1909c4621 100644 --- a/src/wal.h +++ b/src/wal.h @@ -136,7 +136,7 @@ int sqlite3WalSnapshotRecover(Wal *pWal); #ifndef SQLITE_OMIT_CONCURRENT /* Tell the wal layer that we want to commit a concurrent transaction */ -int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPg, Bitvec *pRead); +int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPg, Bitvec *pRead, Pgno*); /* Upgrade the state of the client to take into account changes written ** by other connections */ diff --git a/test/concurrent5.test b/test/concurrent5.test index e77c794048..dde661ea8b 100644 --- a/test/concurrent5.test +++ b/test/concurrent5.test @@ -59,7 +59,7 @@ do_test 1.1.1 { catchsql COMMIT db2 } {1 {database is locked}} do_test_conflict_msg 1.1.2 { - conflict at page 2 (read-only page; part of b-tree with root page 2) + conflict at page 2 (read-only page; part of db table t1) } do_test 1.2.1 { @@ -78,7 +78,7 @@ do_test 1.2.1 { } {1 {database is locked}} do_test_conflict_msg 1.2.2 { - conflict at page 105 (read/write page; part of b-tree with root page 2) + conflict at page 105 (read/write page; part of db table t1) } do_test 1.3.1 { @@ -97,7 +97,31 @@ do_test 1.3.1 { } {1 {database is locked}} do_test_conflict_msg 1.3.2 { - conflict at page 3 (read/write page; part of b-tree with root page 3) + conflict at page 3 (read/write page; part of db table t2) +} + +do_test 1.4.1 { + set ::log [list] + + execsql { + ROLLBACK; + CREATE TABLE t3(a INTEGER PRIMARY KEY, b INTEGER); + CREATE INDEX i3 ON t3(b); + + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<5000 + ) INSERT INTO t3 SELECT i, i FROM s; + + BEGIN CONCURRENT; + INSERT INTO t3 VALUES(0, 5001); + } db2 + + execsql { INSERT INTO t3 VALUES(NULL, 5002) } db + catchsql COMMIT db2 +} {1 {database is locked}} + +do_test_conflict_msg 1.3.2 { + conflict at page 211 (read/write page; part of db index t3.i3) } db close From 7f002db7e3386448e0dc96e9c586f0c3f8501504 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 31 May 2017 17:06:13 +0000 Subject: [PATCH 054/179] Generate extra log messages in response to irregularites in the pointer-map used by "BEGIN CONCURRENT" transactions. FossilOrigin-Name: f7e3e2bc88f110d9282ce5d2fa58580c585faeb57cb707253f05001e5f4bd91b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 38 ++++++++++++++++++++++++++++++++++++++ test/concurrent3.test | 4 ++-- 4 files changed, 48 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 7d09c7ae3f..7df12fad09 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Instead\sof\sa\sroot\spage\snumber,\slog\sthe\sobject\s(table\sor\sindex)\sname\sif\sa\spage\nlevel\slocking\sconflict\sis\sdetected. -D 2017-05-29T19:23:56.135 +C Generate\sextra\slog\smessages\sin\sresponse\sto\sirregularites\sin\sthe\spointer-map\nused\sby\s"BEGIN\sCONCURRENT"\stransactions. +D 2017-05-31T17:06:13.603 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 8eeb80162074004e906b53d7340a12a14c471a83743aab975947e95ce061efcc @@ -350,7 +350,7 @@ F src/auth.c 79f96c6f33bf0e5da8d1c282cee5ebb1852bb8a6ccca3e485d7c459b035d9c3c F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c c77b7f5759e413c1c8b53267d633c952e66db79c1171964c7e24c0f92f5019cf F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 8be55b3e9ed1346b5cfd3f749e6c338ced1cade9242a3c5404696bc4aed26652 +F src/btree.c 322cd2e8aa98c5e44fb9fc756e15fe86c093bdba2e8c4465541098a56b1dd0a7 F src/btree.h 14e99cc2b666beb60322173c761d16b668ec2e07c18bbb74e8a49fe85946f8a0 F src/btreeInt.h 7429915fc8f51bbd78b7ac023aa4afbe5b9660fc1e6970f144b07540a34a4623 F src/build.c ba3f389668754c407805bbc5f8ab140f063ba6b04a6a86f63006b63b3c7319a8 @@ -611,7 +611,7 @@ F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b F test/concfault.test 500f17c3fcfe7705114422bcc6ddd3c740001a43 F test/concurrent.test 3eb5e6a911dc6ff72e3a679f563e683b436f6c701e6e1d6050173df2b8448d6b F test/concurrent2.test 9dfbeb0a323733fe1d13443371734bb94a674dbf777f464365475903873111f8 -F test/concurrent3.test 0a5f7e3036d1eccf0782d7153ac21f5f222e9468 +F test/concurrent3.test f4af1cf1220908c6dd5694923621c19e999b78cd997e2646285f08a52bcb4170 F test/concurrent4.test 989c6575225f9c4ef5d2392a9b9d0405665567c7501a3e44129598794d9b1b5f F test/concurrent5.test d5d7d9d404a9b4502464fc097c1fc5c3012bb4f1b063fae7ad707ca983fc86c5 F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db @@ -1591,7 +1591,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 92618492b048867af38922825f3d094eaaa2dd919b1ed2f7372483cc53f892bf -R 9903c08299ff258c04c9dfdcdc90aac6 +P 9ad846e57bd427adc7c29768cabca18905f7f978168e0642a5917d894fda8bfd +R 524ad4cfe044e13975f9c2307068a2ae U dan -Z 4393c020077681d651727b37beafcbae +Z b8c7e688cb5faa52956d652176272c93 diff --git a/manifest.uuid b/manifest.uuid index cee4fdfd92..063e7cf439 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ad846e57bd427adc7c29768cabca18905f7f978168e0642a5917d894fda8bfd \ No newline at end of file +f7e3e2bc88f110d9282ce5d2fa58580c585faeb57cb707253f05001e5f4bd91b \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 38dfbccf41..183e3db17c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -629,11 +629,45 @@ static void btreePtrmapDelete(BtShared *pBt){ pBt->pMap = 0; } } + +static void btreeCheckPtrmap(BtShared *p, int nPage, const char *zLog){ + BtreePtrmap *pMap = p->pMap; + if( pMap ){ + int n = MIN(1 + nPage - (int)pMap->iFirst, 5); + int i; + for(i=0; iaPtr[i].eType; + if( (eType==PTRMAP_OVERFLOW1 || + eType==PTRMAP_OVERFLOW2 || + eType==PTRMAP_BTREE) && pMap->aPtr[i].parent==0 + ){ + sqlite3_log(SQLITE_ERROR, + "Bitvec: error at (%s) - (%d/%d %d/%d %d/%d %d/%d %d/%d)", + zLog, + (int)pMap->aPtr[0].eType, (int)pMap->aPtr[0].parent, + (n>1 ? (int)pMap->aPtr[1].eType : -1), + (n>1 ? (int)pMap->aPtr[1].parent : -1), + + (n>2 ? (int)pMap->aPtr[2].eType : -1), + (n>2 ? (int)pMap->aPtr[2].parent : -1), + + (n>3 ? (int)pMap->aPtr[3].eType : -1), + (n>3 ? (int)pMap->aPtr[3].parent : -1), + + (n>4 ? (int)pMap->aPtr[4].eType : -1), + (n>4 ? (int)pMap->aPtr[4].parent : -1) + ); + break; + } + } + } +} #else /* SQLITE_OMIT_CONCURRENT */ # define btreePtrmapAllocate(x) SQLITE_OK # define btreePtrmapDelete(x) # define btreePtrmapBegin(x,y) SQLITE_OK # define btreePtrmapEnd(x,y,z) +# define btreeCheckPtrmap(a,b,c) #endif /* SQLITE_OMIT_CONCURRENT */ static void releasePage(MemPage *pPage); /* Forward reference */ @@ -4088,6 +4122,8 @@ static int btreeFixUnlocked(Btree *p){ Pgno nPage = btreePagecount(pBt); u32 nFree = get4byte(&p1[36]); + btreeCheckPtrmap(pBt, nPage, "btreeFixUnlocked(1)"); + assert( pBt->pMap ); rc = sqlite3PagerUpgradeSnapshot(pPager, pPage1->pDbPage); assert( p1==pPage1->aData ); @@ -8674,6 +8710,7 @@ int sqlite3BtreeInsert( assert( pCur->apPage[pCur->iPage]->nOverflow==0 ); end_insert: + btreeCheckPtrmap(pBt, pBt->nPage, "sqlite3BtreeInsert()"); return rc; } @@ -8845,6 +8882,7 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ } } } + btreeCheckPtrmap(pBt, pBt->nPage, "sqlite3BtreeInsert()"); return rc; } diff --git a/test/concurrent3.test b/test/concurrent3.test index 3ad6a1cce4..4364f84012 100644 --- a/test/concurrent3.test +++ b/test/concurrent3.test @@ -26,8 +26,8 @@ ifcapable !concurrent { db close sqlite3_shutdown -#test_sqlite3_log xLog -#proc xLog {error_code msg} { puts "$error_code: $msg" } +test_sqlite3_log xLog +proc xLog {error_code msg} { puts "$error_code: $msg" } reset_db proc create_schema {} { From 35dcd0e89231faa9af59bc24edafbd09dd28d240 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 2 Jun 2017 09:31:27 +0000 Subject: [PATCH 055/179] Extend even further the logging designed to find problems in the pointer-map structure. Call abort() to dump a core as soon as such a problem is seen. FossilOrigin-Name: f131677dcb4937e0dd62626afa91756aa28079e92acd6e9e127f6f676aa334f9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 10 +++++++++- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 7df12fad09..7622016606 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Generate\sextra\slog\smessages\sin\sresponse\sto\sirregularites\sin\sthe\spointer-map\nused\sby\s"BEGIN\sCONCURRENT"\stransactions. -D 2017-05-31T17:06:13.603 +C Extend\seven\sfurther\sthe\slogging\sdesigned\sto\sfind\sproblems\sin\sthe\spointer-map\nstructure.\sCall\sabort()\sto\sdump\sa\score\sas\ssoon\sas\ssuch\sa\sproblem\sis\sseen. +D 2017-06-02T09:31:27.185 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 8eeb80162074004e906b53d7340a12a14c471a83743aab975947e95ce061efcc @@ -350,7 +350,7 @@ F src/auth.c 79f96c6f33bf0e5da8d1c282cee5ebb1852bb8a6ccca3e485d7c459b035d9c3c F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c c77b7f5759e413c1c8b53267d633c952e66db79c1171964c7e24c0f92f5019cf F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 322cd2e8aa98c5e44fb9fc756e15fe86c093bdba2e8c4465541098a56b1dd0a7 +F src/btree.c 63ae7953340839b84a1462f01414debebf8a178867085189280698f0cf724219 F src/btree.h 14e99cc2b666beb60322173c761d16b668ec2e07c18bbb74e8a49fe85946f8a0 F src/btreeInt.h 7429915fc8f51bbd78b7ac023aa4afbe5b9660fc1e6970f144b07540a34a4623 F src/build.c ba3f389668754c407805bbc5f8ab140f063ba6b04a6a86f63006b63b3c7319a8 @@ -1591,7 +1591,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9ad846e57bd427adc7c29768cabca18905f7f978168e0642a5917d894fda8bfd -R 524ad4cfe044e13975f9c2307068a2ae +P f7e3e2bc88f110d9282ce5d2fa58580c585faeb57cb707253f05001e5f4bd91b +R 0cc4cace2096d9ccd320cd1d9a64685a U dan -Z b8c7e688cb5faa52956d652176272c93 +Z 40b70ad4ffb9dd03aeca52705a79b953 diff --git a/manifest.uuid b/manifest.uuid index 063e7cf439..c8b24c331b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f7e3e2bc88f110d9282ce5d2fa58580c585faeb57cb707253f05001e5f4bd91b \ No newline at end of file +f131677dcb4937e0dd62626afa91756aa28079e92acd6e9e127f6f676aa334f9 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 183e3db17c..e495aa3297 100644 --- a/src/btree.c +++ b/src/btree.c @@ -657,6 +657,7 @@ static void btreeCheckPtrmap(BtShared *p, int nPage, const char *zLog){ (n>4 ? (int)pMap->aPtr[4].eType : -1), (n>4 ? (int)pMap->aPtr[4].parent : -1) ); + abort(); break; } } @@ -8806,6 +8807,8 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ invalidateIncrblobCursors(p, pCur->pgnoRoot, pCur->info.nKey, 0); } + btreeCheckPtrmap(pBt, pBt->nPage, "sqlite3BtreeDelete(0)"); + /* Make the page containing the entry to be deleted writable. Then free any ** overflow pages associated with the entry and finally remove the cell ** itself from within the page. */ @@ -8815,6 +8818,8 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ dropCell(pPage, iCellIdx, info.nSize, &rc); if( rc ) return rc; + btreeCheckPtrmap(pBt, pBt->nPage, "sqlite3BtreeDelete(1)"); + /* If the cell deleted was not located on a leaf page, then the cursor ** is currently pointing to the largest entry in the sub-tree headed ** by the child-page of the cell that was just deleted from an internal @@ -8840,6 +8845,8 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ if( rc ) return rc; } + btreeCheckPtrmap(pBt, pBt->nPage, "sqlite3BtreeDelete(2)"); + /* Balance the tree. If the entry deleted was located on a leaf page, ** then the cursor still points to that page. In this case the first ** call to balance() repairs the tree, and the if(...) condition is @@ -8863,6 +8870,8 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ rc = balance(pCur); } + btreeCheckPtrmap(pBt, pBt->nPage, "sqlite3BtreeDelete(3)"); + if( rc==SQLITE_OK ){ if( bSkipnext ){ assert( bPreserve && (pCur->iPage==iCellDepth || CORRUPT_DB) ); @@ -8882,7 +8891,6 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ } } } - btreeCheckPtrmap(pBt, pBt->nPage, "sqlite3BtreeInsert()"); return rc; } From 50179f91f391d4b65b192f9495ea221ca8a9ff90 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 8 Jun 2017 11:26:13 +0000 Subject: [PATCH 056/179] Ensure pointer map entries are always added when a row that does use overflow pages replaces one that does not in an auto-vacuum database. Fix for [fda22108]. FossilOrigin-Name: 9478106ca963e3ae5cfe59da40301c5a0a07494d03e656b5eb10ab45e441b870 --- manifest | 18 ++++++++---------- manifest.uuid | 2 +- src/btree.c | 10 ++++++++-- test/autovacuum.test | 7 +++++++ 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index d8ee2f52b1..7448590647 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Version\s3.19.2 -D 2017-05-25T16:50:27.649 +C Ensure\spointer\smap\sentries\sare\salways\sadded\swhen\sa\srow\sthat\sdoes\suse\soverflow\npages\sreplaces\sone\sthat\sdoes\snot\sin\san\sauto-vacuum\sdatabase.\sFix\sfor\n[fda22108]. +D 2017-06-08T11:26:13.795 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 8eeb80162074004e906b53d7340a12a14c471a83743aab975947e95ce061efcc @@ -346,7 +346,7 @@ F src/auth.c 79f96c6f33bf0e5da8d1c282cee5ebb1852bb8a6ccca3e485d7c459b035d9c3c F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 8c1fd4cfa2b0bf021386e0a1f4e30b64eea7a2c1bc2e0c3e5901a626b1ab6aa9 +F src/btree.c de57bf3f205082eb4ad8c923af7ecd1d10c4f15656f6a38038a3bd9b8ac3fc2a F src/btree.h 80f518c0788be6cec8d9f8e13bd8e380df299d2b5e4ac340dc887b0642647cfc F src/btreeInt.h a392d353104b4add58b4a59cb185f5d5693dde832c565b77d8d4c343ed98f610 F src/build.c 4026a9c554b233e50c5e9ad46963e676cf54dd2306d952aa1eaa07a1bc9ce14f @@ -537,7 +537,7 @@ F test/autoindex2.test 12ef578928102baaa0dc23ad397601a2f4ecb0df F test/autoindex3.test a3be0d1a53a7d2edff208a5e442312957047e972 F test/autoindex4.test 49d3cd791a9baa16fb461d7ea3de80d019a819cf F test/autoindex5.test 96f084a5e6024ea07cace5888df3223f3ea86990 -F test/autovacuum.test 92c24eedbdb68e49f3fb71f26f9ce6d8988cac15 +F test/autovacuum.test 0831cd34e14695d297187f7f6519265e3121c5b0a1720e548e86829e796129e9 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85 F test/backcompat.test 3e64cedda754c778ef6bbe417b6e7a295e662a4d @@ -1580,10 +1580,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1a074c8a2bc0b28918ef905339d11a21d30101b4ea8c06c8b3faca7d17237538 -R f4b3c258ec4a8c30fd1362752bfd9b5c -T +bgcolor * #d0c0ff -T +sym-release * -T +sym-version-3.19.2 * +P edb4e819b0c058c7d74d27ebd14cc5ceb2bad6a6144a486a970182b7afe3f8b9 +Q +b30dfba811cb531b09ff2e71a1a18ed53c816cb39155dd52ca3e2701425fe17b +R f3c8c8383c25bcc4e304081dde6d0ab0 U drh -Z 1c4850e8eeff11c8d32e6493160f83cd +Z f97aca8e1bd3663fbf75118ceaa954dd diff --git a/manifest.uuid b/manifest.uuid index b316b7a155..981f2490fd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -edb4e819b0c058c7d74d27ebd14cc5ceb2bad6a6144a486a970182b7afe3f8b9 \ No newline at end of file +9478106ca963e3ae5cfe59da40301c5a0a07494d03e656b5eb10ab45e441b870 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index e5edf44c4d..1f9a6e4872 100644 --- a/src/btree.c +++ b/src/btree.c @@ -8174,12 +8174,18 @@ int sqlite3BtreeInsert( memcpy(newCell, oldCell, 4); } rc = clearCell(pPage, oldCell, &info); - if( info.nSize==szNew && info.nLocal==info.nPayload ){ + if( info.nSize==szNew && info.nLocal==info.nPayload + && (!ISAUTOVACUUM || szNewminLocal) + ){ /* Overwrite the old cell with the new if they are the same size. ** We could also try to do this if the old cell is smaller, then add ** the leftover space to the free list. But experiments show that ** doing that is no faster then skipping this optimization and just - ** calling dropCell() and insertCell(). */ + ** calling dropCell() and insertCell(). + ** + ** This optimization cannot be used on an autovacuum database if the + ** new entry uses overflow pages, as the insertCell() call below is + ** necessary to add the PTRMAP_OVERFLOW1 pointer-map entry. */ assert( rc==SQLITE_OK ); /* clearCell never fails when nLocal==nPayload */ if( oldCell+szNew > pPage->aDataEnd ) return SQLITE_CORRUPT_BKPT; memcpy(oldCell, newCell, szNew); diff --git a/test/autovacuum.test b/test/autovacuum.test index 5c14ed85d3..431c4b8a2a 100644 --- a/test/autovacuum.test +++ b/test/autovacuum.test @@ -705,5 +705,12 @@ do_test autovacuum-9.5 { file size test.db } $::sqlite_pending_byte +do_execsql_test autovacuum-10.1 { + DROP TABLE t1; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(25, randomblob(104)); + REPLACE INTO t1 VALUES(25, randomblob(1117)); + PRAGMA integrity_check; +} {ok} finish_test From a096fcc73c82f17e4194089f9b3f4c8a224781c7 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 8 Jun 2017 11:27:05 +0000 Subject: [PATCH 057/179] Increase the version number to 3.19.3. FossilOrigin-Name: 903fff53b3f02cbc011c4cd47d841f45a447746fdf3bcb01202e27c3f2b57589 --- VERSION | 2 +- configure | 18 +++++++++--------- manifest | 15 +++++++-------- manifest.uuid | 2 +- 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/VERSION b/VERSION index a8346d5d11..0b3135213f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.19.2 +3.19.3 diff --git a/configure b/configure index 653accad11..8c672fadce 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for sqlite 3.19.2. +# Generated by GNU Autoconf 2.69 for sqlite 3.19.3. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -726,8 +726,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.19.2' -PACKAGE_STRING='sqlite 3.19.2' +PACKAGE_VERSION='3.19.3' +PACKAGE_STRING='sqlite 3.19.3' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1463,7 +1463,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.19.2 to adapt to many kinds of systems. +\`configure' configures sqlite 3.19.3 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1528,7 +1528,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.19.2:";; + short | recursive ) echo "Configuration of sqlite 3.19.3:";; esac cat <<\_ACEOF @@ -1652,7 +1652,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.19.2 +sqlite configure 3.19.3 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2071,7 +2071,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.19.2, which was +It was created by sqlite $as_me 3.19.3, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -12151,7 +12151,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.19.2, which was +This file was extended by sqlite $as_me 3.19.3, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -12217,7 +12217,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -sqlite config.status 3.19.2 +sqlite config.status 3.19.3 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/manifest b/manifest index 7448590647..81321a8b5d 100644 --- a/manifest +++ b/manifest @@ -1,10 +1,10 @@ -C Ensure\spointer\smap\sentries\sare\salways\sadded\swhen\sa\srow\sthat\sdoes\suse\soverflow\npages\sreplaces\sone\sthat\sdoes\snot\sin\san\sauto-vacuum\sdatabase.\sFix\sfor\n[fda22108]. -D 2017-06-08T11:26:13.795 +C Increase\sthe\sversion\snumber\sto\s3.19.3. +D 2017-06-08T11:27:05.632 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 8eeb80162074004e906b53d7340a12a14c471a83743aab975947e95ce061efcc F README.md 2b15fae33852f2f53996774c21fb41e1d94181c4401a0e43ac93e11f2cc901b9 -F VERSION 10a1d396b8a19db3c3cc6c7b909be2cf7ad97c517b4f00af363506fd85aab713 +F VERSION 6035ef7e627075d9a9194304c3a07acb2736a0abd0dd980a7ea75a3a45719e86 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 @@ -30,7 +30,7 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 6376abec766e9a0785178b1823b5a587e9f1ccbc F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure 1b0309062481dbde6278cb870f1680c81c65202d0bfde469f81c7e22397d6f23 x +F configure 51470e605be2083c8ac6e7973a1860ead8be59562f2121197216b70caa0d8b9d x F configure.ac 901e8db2c211e8b655e51ee24f2b8faa33407acde87effc47f70380f3b04e452 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html b5a3c07d33ecb8e019ce8f7660fe2dbbad9d7977 @@ -1580,8 +1580,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P edb4e819b0c058c7d74d27ebd14cc5ceb2bad6a6144a486a970182b7afe3f8b9 -Q +b30dfba811cb531b09ff2e71a1a18ed53c816cb39155dd52ca3e2701425fe17b -R f3c8c8383c25bcc4e304081dde6d0ab0 +P 9478106ca963e3ae5cfe59da40301c5a0a07494d03e656b5eb10ab45e441b870 +R 28460bdf0816a2fdb51bf300e77b3848 U drh -Z f97aca8e1bd3663fbf75118ceaa954dd +Z 52bfa7329c4a7a037c3124284cb2a054 diff --git a/manifest.uuid b/manifest.uuid index 981f2490fd..057626a2f4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9478106ca963e3ae5cfe59da40301c5a0a07494d03e656b5eb10ab45e441b870 \ No newline at end of file +903fff53b3f02cbc011c4cd47d841f45a447746fdf3bcb01202e27c3f2b57589 \ No newline at end of file From d4d01a689d6f41c324386dc7735d60e3f1480b35 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 8 Jun 2017 11:32:19 +0000 Subject: [PATCH 058/179] Fix an issue with OPT_FEATURE_FLAGS in configure.ac. FossilOrigin-Name: 97b5c4a53d0af3792c5ae5ceafdb02245f841c2bd4d79bcfd0ffdf6c5ff27c9e --- configure | 20 ++++++++++---------- configure.ac | 20 ++++++++++---------- manifest | 15 ++++++++------- manifest.uuid | 2 +- 4 files changed, 29 insertions(+), 28 deletions(-) diff --git a/configure b/configure index 8c672fadce..ee7ae2a121 100755 --- a/configure +++ b/configure @@ -11356,7 +11356,7 @@ fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS5" >&5 $as_echo_n "checking whether to support MEMSYS5... " >&6; } if test "${enable_memsys5}" = "yes"; then - OPT_FEATURE_FLAGS="$(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_MEMSYS5" + OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS5" { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } else @@ -11373,7 +11373,7 @@ fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS3" >&5 $as_echo_n "checking whether to support MEMSYS3... " >&6; } if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then - OPT_FEATURE_FLAGS="$(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_MEMSYS3" + OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS3" { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } else @@ -11391,7 +11391,7 @@ else fi if test "${enable_fts3}" = "yes" ; then - OPT_FEATURE_FLAGS="$(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3" + OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS3" fi # Check whether --enable-fts4 was given. if test "${enable_fts4+set}" = set; then : @@ -11401,7 +11401,7 @@ else fi if test "${enable_fts4}" = "yes" ; then - OPT_FEATURE_FLAGS="$(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS4" + OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS4" { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing log" >&5 $as_echo_n "checking for library containing log... " >&6; } if ${ac_cv_search_log+:} false; then : @@ -11467,7 +11467,7 @@ else fi if test "${enable_fts5}" = "yes" ; then - OPT_FEATURE_FLAGS="$(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS5" + OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS5" { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing log" >&5 $as_echo_n "checking for library containing log... " >&6; } if ${ac_cv_search_log+:} false; then : @@ -11536,7 +11536,7 @@ else fi if test "${enable_json1}" = "yes" ; then - OPT_FEATURE_FLAGS="$(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_JSON1" + OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_JSON1" fi ######### @@ -11549,7 +11549,7 @@ else fi if test "${enable_rtree}" = "yes" ; then - OPT_FEATURE_FLAGS="$(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE" + OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_RTREE" fi ######### @@ -11562,12 +11562,12 @@ else fi if test "${enable_session}" = "yes" ; then - OPT_FEATURE_FLAGS="$(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_SESSION" - OPT_FEATURE_FLAGS="$(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PREUPDATE_HOOK" + OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_SESSION" + OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_PREUPDATE_HOOK" fi ######### -# attempt to duplicate any OMITS and ENABLES into the $(OPT_FEATURE_FLAGS) parameter +# attempt to duplicate any OMITS and ENABLES into the ${OPT_FEATURE_FLAGS} parameter for option in $CFLAGS $CPPFLAGS do case $option in diff --git a/configure.ac b/configure.ac index a0aeb94f72..4deee8ddcb 100644 --- a/configure.ac +++ b/configure.ac @@ -596,7 +596,7 @@ AC_ARG_ENABLE(memsys5, [enable_memsys5=yes],[enable_memsys5=no]) AC_MSG_CHECKING([whether to support MEMSYS5]) if test "${enable_memsys5}" = "yes"; then - OPT_FEATURE_FLAGS="$(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_MEMSYS5" + OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS5" AC_MSG_RESULT([yes]) else AC_MSG_RESULT([no]) @@ -606,7 +606,7 @@ AC_ARG_ENABLE(memsys3, [enable_memsys3=yes],[enable_memsys3=no]) AC_MSG_CHECKING([whether to support MEMSYS3]) if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then - OPT_FEATURE_FLAGS="$(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_MEMSYS3" + OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS3" AC_MSG_RESULT([yes]) else AC_MSG_RESULT([no]) @@ -618,20 +618,20 @@ AC_ARG_ENABLE(fts3, AC_HELP_STRING([--enable-fts3], [Enable the FTS3 extension]), [enable_fts3=yes],[enable_fts3=no]) if test "${enable_fts3}" = "yes" ; then - OPT_FEATURE_FLAGS="$(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3" + OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS3" fi AC_ARG_ENABLE(fts4, AC_HELP_STRING([--enable-fts4], [Enable the FTS4 extension]), [enable_fts4=yes],[enable_fts4=no]) if test "${enable_fts4}" = "yes" ; then - OPT_FEATURE_FLAGS="$(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS4" + OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS4" AC_SEARCH_LIBS([log],[m]) fi AC_ARG_ENABLE(fts5, AC_HELP_STRING([--enable-fts5], [Enable the FTS5 extension]), [enable_fts5=yes],[enable_fts5=no]) if test "${enable_fts5}" = "yes" ; then - OPT_FEATURE_FLAGS="$(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS5" + OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS5" AC_SEARCH_LIBS([log],[m]) fi @@ -641,7 +641,7 @@ AC_ARG_ENABLE(json1, AC_HELP_STRING([--enable-json1], [Enable the JSON1 extension]), [enable_json1=yes],[enable_json1=no]) if test "${enable_json1}" = "yes" ; then - OPT_FEATURE_FLAGS="$(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_JSON1" + OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_JSON1" fi ######### @@ -650,7 +650,7 @@ AC_ARG_ENABLE(rtree, AC_HELP_STRING([--enable-rtree], [Enable the RTREE extension]), [enable_rtree=yes],[enable_rtree=no]) if test "${enable_rtree}" = "yes" ; then - OPT_FEATURE_FLAGS="$(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE" + OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_RTREE" fi ######### @@ -659,12 +659,12 @@ AC_ARG_ENABLE(session, AC_HELP_STRING([--enable-session], [Enable the SESSION extension]), [enable_session=yes],[enable_session=no]) if test "${enable_session}" = "yes" ; then - OPT_FEATURE_FLAGS="$(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_SESSION" - OPT_FEATURE_FLAGS="$(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PREUPDATE_HOOK" + OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_SESSION" + OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_PREUPDATE_HOOK" fi ######### -# attempt to duplicate any OMITS and ENABLES into the $(OPT_FEATURE_FLAGS) parameter +# attempt to duplicate any OMITS and ENABLES into the ${OPT_FEATURE_FLAGS} parameter for option in $CFLAGS $CPPFLAGS do case $option in diff --git a/manifest b/manifest index 81321a8b5d..f58be9d0ea 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\sversion\snumber\sto\s3.19.3. -D 2017-06-08T11:27:05.632 +C Fix\san\sissue\swith\sOPT_FEATURE_FLAGS\sin\sconfigure.ac. +D 2017-06-08T11:32:19.653 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 8eeb80162074004e906b53d7340a12a14c471a83743aab975947e95ce061efcc @@ -30,8 +30,8 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 6376abec766e9a0785178b1823b5a587e9f1ccbc F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure 51470e605be2083c8ac6e7973a1860ead8be59562f2121197216b70caa0d8b9d x -F configure.ac 901e8db2c211e8b655e51ee24f2b8faa33407acde87effc47f70380f3b04e452 +F configure 1a94740b2ac20f3b865689d32746ade6eee7974282170b0ba2b019d310917ec7 x +F configure.ac 13f45f02e6c51dd0e347315b5401c3f047712b7f79b7f35619115c23755afcff F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html b5a3c07d33ecb8e019ce8f7660fe2dbbad9d7977 F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710 @@ -1580,7 +1580,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9478106ca963e3ae5cfe59da40301c5a0a07494d03e656b5eb10ab45e441b870 -R 28460bdf0816a2fdb51bf300e77b3848 +P 903fff53b3f02cbc011c4cd47d841f45a447746fdf3bcb01202e27c3f2b57589 +Q +43ce3bd3a4b41ea757e308e563d6c1bf7f9c09f10e4766c536d0bf1042c45613 +R b94272eb5bee6b36b4ac20fd561bc415 U drh -Z 52bfa7329c4a7a037c3124284cb2a054 +Z 9d46edb93ef6aa9b74fdbf7152b194d5 diff --git a/manifest.uuid b/manifest.uuid index 057626a2f4..f110d18634 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -903fff53b3f02cbc011c4cd47d841f45a447746fdf3bcb01202e27c3f2b57589 \ No newline at end of file +97b5c4a53d0af3792c5ae5ceafdb02245f841c2bd4d79bcfd0ffdf6c5ff27c9e \ No newline at end of file From d6396e9b169ccedcd9e8ba07f6ecba68d558e61b Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 8 Jun 2017 14:26:16 +0000 Subject: [PATCH 059/179] Version 3.19.3 FossilOrigin-Name: 0ee482a1e0eae22e08edc8978c9733a96603d4509645f348ebf55b579e89636b --- manifest | 12 +++++++----- manifest.uuid | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/manifest b/manifest index f58be9d0ea..5b7be2b7f8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sissue\swith\sOPT_FEATURE_FLAGS\sin\sconfigure.ac. -D 2017-06-08T11:32:19.653 +C Version\s3.19.3 +D 2017-06-08T14:26:16.283 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 8eeb80162074004e906b53d7340a12a14c471a83743aab975947e95ce061efcc @@ -1580,8 +1580,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 903fff53b3f02cbc011c4cd47d841f45a447746fdf3bcb01202e27c3f2b57589 -Q +43ce3bd3a4b41ea757e308e563d6c1bf7f9c09f10e4766c536d0bf1042c45613 +P 97b5c4a53d0af3792c5ae5ceafdb02245f841c2bd4d79bcfd0ffdf6c5ff27c9e R b94272eb5bee6b36b4ac20fd561bc415 +T +bgcolor * #d0c0ff +T +sym-release * +T +sym-version-3.19.3 * U drh -Z 9d46edb93ef6aa9b74fdbf7152b194d5 +Z 0ee14e0381c757607413d6046358676f diff --git a/manifest.uuid b/manifest.uuid index f110d18634..fd64fbcbc1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -97b5c4a53d0af3792c5ae5ceafdb02245f841c2bd4d79bcfd0ffdf6c5ff27c9e \ No newline at end of file +0ee482a1e0eae22e08edc8978c9733a96603d4509645f348ebf55b579e89636b \ No newline at end of file From 4956bd5f9aa5b14807d175b109819a5be9972c92 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 8 Jun 2017 16:23:55 +0000 Subject: [PATCH 060/179] Update the recent auto-vacuum fix so that it works for the in-memory pointer-map structure used by this branch. FossilOrigin-Name: 8e311a6dba202e8733830d8f31b8f0ce11eaefb3a0ab5e5e95ac0d2e5136043b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 2 +- test/concurrent4.test | 16 ++++++++++++++++ 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index f9144add04..6d09c0d308 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\s3.19.3\schanges\swith\sthis\sbranch. -D 2017-06-08T16:10:41.913 +C Update\sthe\srecent\sauto-vacuum\sfix\sso\sthat\sit\sworks\sfor\sthe\sin-memory\npointer-map\sstructure\sused\sby\sthis\sbranch. +D 2017-06-08T16:23:55.894 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 8eeb80162074004e906b53d7340a12a14c471a83743aab975947e95ce061efcc @@ -350,7 +350,7 @@ F src/auth.c 79f96c6f33bf0e5da8d1c282cee5ebb1852bb8a6ccca3e485d7c459b035d9c3c F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c c77b7f5759e413c1c8b53267d633c952e66db79c1171964c7e24c0f92f5019cf F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 95dd5106cb3ee133610b8fab3d1ab1d76ae7f9ef3fae11d023836dc3c672d2b7 +F src/btree.c 1394abd656dab0f62cfe7060be026451bb7fb75a2c5f5a2ea95484264d105614 F src/btree.h 14e99cc2b666beb60322173c761d16b668ec2e07c18bbb74e8a49fe85946f8a0 F src/btreeInt.h 7429915fc8f51bbd78b7ac023aa4afbe5b9660fc1e6970f144b07540a34a4623 F src/build.c ba3f389668754c407805bbc5f8ab140f063ba6b04a6a86f63006b63b3c7319a8 @@ -612,7 +612,7 @@ F test/concfault.test 500f17c3fcfe7705114422bcc6ddd3c740001a43 F test/concurrent.test 3eb5e6a911dc6ff72e3a679f563e683b436f6c701e6e1d6050173df2b8448d6b F test/concurrent2.test 9dfbeb0a323733fe1d13443371734bb94a674dbf777f464365475903873111f8 F test/concurrent3.test f4af1cf1220908c6dd5694923621c19e999b78cd997e2646285f08a52bcb4170 -F test/concurrent4.test 989c6575225f9c4ef5d2392a9b9d0405665567c7501a3e44129598794d9b1b5f +F test/concurrent4.test 653de3066911acfb9dcf3802bf4f1981b392b86c11f75e2c38ed1abfdd162293 F test/concurrent5.test d5d7d9d404a9b4502464fc097c1fc5c3012bb4f1b063fae7ad707ca983fc86c5 F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c @@ -1591,7 +1591,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0f59bb94d5e8e16a47c53ef0d6138a35c87230ad384b58008e8e7127780446d8 0ee482a1e0eae22e08edc8978c9733a96603d4509645f348ebf55b579e89636b -R f7ed8b840e4f80adc0a45856c6802bc0 +P e2d38d51a9cf1c3dfef742507ec76e3d35853bd09b0d09bf2d404c4b036a184d +R b03e2900f702f1d9793bda1b10355a73 U dan -Z 8407296cfc564f67ed28c7f44ba3d184 +Z f6db321d9b4a7908768625f45dff8451 diff --git a/manifest.uuid b/manifest.uuid index b3d31c2fb1..1378a871d3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e2d38d51a9cf1c3dfef742507ec76e3d35853bd09b0d09bf2d404c4b036a184d \ No newline at end of file +8e311a6dba202e8733830d8f31b8f0ce11eaefb3a0ab5e5e95ac0d2e5136043b \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 359e699368..75bf0fbdfa 100644 --- a/src/btree.c +++ b/src/btree.c @@ -8638,7 +8638,7 @@ int sqlite3BtreeInsert( } rc = clearCell(pPage, oldCell, &info); if( info.nSize==szNew && info.nLocal==info.nPayload - && (!ISAUTOVACUUM || szNewminLocal) + && (!REQUIRE_PTRMAP || szNewminLocal) ){ /* Overwrite the old cell with the new if they are the same size. ** We could also try to do this if the old cell is smaller, then add diff --git a/test/concurrent4.test b/test/concurrent4.test index ce55cdd848..dd29434381 100644 --- a/test/concurrent4.test +++ b/test/concurrent4.test @@ -84,6 +84,22 @@ do_multiclient_test tn { } {ok} } +reset_db +do_execsql_test 3.1 { + PRAGMA page_size = 1024; + PRAGMA journal_mode = wal; + CREATE TABLE t2(x); + INSERT INTO t2 VALUES(randomblob(5000)); + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(25, randomblob(104)); + DELETE FROM t2; +} {wal} + +do_execsql_test 3.2 { + BEGIN CONCURRENT; + REPLACE INTO t1 VALUES(25, randomblob(1117)); + COMMIT; +} {} finish_test From 827c9b785e7a8ba2a1b80ad69c150747c4d893af Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 10 Jun 2017 17:23:20 +0000 Subject: [PATCH 061/179] Remove sqlite3_log() and abort() calls added to this branch to debug the pointer-map problem ([fda22108]). FossilOrigin-Name: 79544fc2856f30cac8b0962d00698974e8918562f09769a68264d17e1e1176fe --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 46 ---------------------------------------------- 3 files changed, 7 insertions(+), 53 deletions(-) diff --git a/manifest b/manifest index 6d09c0d308..6d77845a0c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\srecent\sauto-vacuum\sfix\sso\sthat\sit\sworks\sfor\sthe\sin-memory\npointer-map\sstructure\sused\sby\sthis\sbranch. -D 2017-06-08T16:23:55.894 +C Remove\ssqlite3_log()\sand\sabort()\scalls\sadded\sto\sthis\sbranch\sto\sdebug\sthe\npointer-map\sproblem\s([fda22108]). +D 2017-06-10T17:23:20.663 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 8eeb80162074004e906b53d7340a12a14c471a83743aab975947e95ce061efcc @@ -350,7 +350,7 @@ F src/auth.c 79f96c6f33bf0e5da8d1c282cee5ebb1852bb8a6ccca3e485d7c459b035d9c3c F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c c77b7f5759e413c1c8b53267d633c952e66db79c1171964c7e24c0f92f5019cf F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 1394abd656dab0f62cfe7060be026451bb7fb75a2c5f5a2ea95484264d105614 +F src/btree.c 5a93ba67ecdcde5e0112c355dffbcb067891c9f2211dbc1e232796e279855de6 F src/btree.h 14e99cc2b666beb60322173c761d16b668ec2e07c18bbb74e8a49fe85946f8a0 F src/btreeInt.h 7429915fc8f51bbd78b7ac023aa4afbe5b9660fc1e6970f144b07540a34a4623 F src/build.c ba3f389668754c407805bbc5f8ab140f063ba6b04a6a86f63006b63b3c7319a8 @@ -1591,7 +1591,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e2d38d51a9cf1c3dfef742507ec76e3d35853bd09b0d09bf2d404c4b036a184d -R b03e2900f702f1d9793bda1b10355a73 +P 8e311a6dba202e8733830d8f31b8f0ce11eaefb3a0ab5e5e95ac0d2e5136043b +R 6be74687e7f9ac02928f68f0a8d639cb U dan -Z f6db321d9b4a7908768625f45dff8451 +Z a53782ebd88a9fbbd1f17403d5393b28 diff --git a/manifest.uuid b/manifest.uuid index 1378a871d3..14de91ea10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8e311a6dba202e8733830d8f31b8f0ce11eaefb3a0ab5e5e95ac0d2e5136043b \ No newline at end of file +79544fc2856f30cac8b0962d00698974e8918562f09769a68264d17e1e1176fe \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 75bf0fbdfa..fadb7eac94 100644 --- a/src/btree.c +++ b/src/btree.c @@ -629,46 +629,11 @@ static void btreePtrmapDelete(BtShared *pBt){ pBt->pMap = 0; } } - -static void btreeCheckPtrmap(BtShared *p, int nPage, const char *zLog){ - BtreePtrmap *pMap = p->pMap; - if( pMap ){ - int n = MIN(1 + nPage - (int)pMap->iFirst, 5); - int i; - for(i=0; iaPtr[i].eType; - if( (eType==PTRMAP_OVERFLOW1 || - eType==PTRMAP_OVERFLOW2 || - eType==PTRMAP_BTREE) && pMap->aPtr[i].parent==0 - ){ - sqlite3_log(SQLITE_ERROR, - "Bitvec: error at (%s) - (%d/%d %d/%d %d/%d %d/%d %d/%d)", - zLog, - (int)pMap->aPtr[0].eType, (int)pMap->aPtr[0].parent, - (n>1 ? (int)pMap->aPtr[1].eType : -1), - (n>1 ? (int)pMap->aPtr[1].parent : -1), - - (n>2 ? (int)pMap->aPtr[2].eType : -1), - (n>2 ? (int)pMap->aPtr[2].parent : -1), - - (n>3 ? (int)pMap->aPtr[3].eType : -1), - (n>3 ? (int)pMap->aPtr[3].parent : -1), - - (n>4 ? (int)pMap->aPtr[4].eType : -1), - (n>4 ? (int)pMap->aPtr[4].parent : -1) - ); - abort(); - break; - } - } - } -} #else /* SQLITE_OMIT_CONCURRENT */ # define btreePtrmapAllocate(x) SQLITE_OK # define btreePtrmapDelete(x) # define btreePtrmapBegin(x,y) SQLITE_OK # define btreePtrmapEnd(x,y,z) -# define btreeCheckPtrmap(a,b,c) #endif /* SQLITE_OMIT_CONCURRENT */ static void releasePage(MemPage *pPage); /* Forward reference */ @@ -4123,8 +4088,6 @@ static int btreeFixUnlocked(Btree *p){ Pgno nPage = btreePagecount(pBt); u32 nFree = get4byte(&p1[36]); - btreeCheckPtrmap(pBt, nPage, "btreeFixUnlocked(1)"); - assert( pBt->pMap ); rc = sqlite3PagerUpgradeSnapshot(pPager, pPage1->pDbPage); assert( p1==pPage1->aData ); @@ -8717,7 +8680,6 @@ int sqlite3BtreeInsert( assert( pCur->apPage[pCur->iPage]->nOverflow==0 ); end_insert: - btreeCheckPtrmap(pBt, pBt->nPage, "sqlite3BtreeInsert()"); return rc; } @@ -8813,8 +8775,6 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ invalidateIncrblobCursors(p, pCur->pgnoRoot, pCur->info.nKey, 0); } - btreeCheckPtrmap(pBt, pBt->nPage, "sqlite3BtreeDelete(0)"); - /* Make the page containing the entry to be deleted writable. Then free any ** overflow pages associated with the entry and finally remove the cell ** itself from within the page. */ @@ -8824,8 +8784,6 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ dropCell(pPage, iCellIdx, info.nSize, &rc); if( rc ) return rc; - btreeCheckPtrmap(pBt, pBt->nPage, "sqlite3BtreeDelete(1)"); - /* If the cell deleted was not located on a leaf page, then the cursor ** is currently pointing to the largest entry in the sub-tree headed ** by the child-page of the cell that was just deleted from an internal @@ -8851,8 +8809,6 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ if( rc ) return rc; } - btreeCheckPtrmap(pBt, pBt->nPage, "sqlite3BtreeDelete(2)"); - /* Balance the tree. If the entry deleted was located on a leaf page, ** then the cursor still points to that page. In this case the first ** call to balance() repairs the tree, and the if(...) condition is @@ -8876,8 +8832,6 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ rc = balance(pCur); } - btreeCheckPtrmap(pBt, pBt->nPage, "sqlite3BtreeDelete(3)"); - if( rc==SQLITE_OK ){ if( bSkipnext ){ assert( bPreserve && (pCur->iPage==iCellDepth || CORRUPT_DB) ); From 7365bcd72c17412b9a9a59eccd58e01ad01dc6b0 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Jul 2017 18:28:33 +0000 Subject: [PATCH 062/179] Get the build working with SQLITE_OMIT_CONCURRENT. FossilOrigin-Name: a29401e924e6f82ca1c3eec98c6c0b2f1d5b9d7fba9af41a78516f21a7bab3a2 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 5 ++++- src/btreeInt.h | 6 +++--- src/pager.h | 2 ++ 5 files changed, 18 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 96896f8708..6ae55d542a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\sthe\slatest\strunk\senhancements. -D 2017-07-20T17:47:12.460 +C Get\sthe\sbuild\sworking\swith\sSQLITE_OMIT_CONCURRENT. +D 2017-07-20T18:28:33.979 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016 @@ -399,9 +399,9 @@ F src/auth.c 79f96c6f33bf0e5da8d1c282cee5ebb1852bb8a6ccca3e485d7c459b035d9c3c F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c c77b7f5759e413c1c8b53267d633c952e66db79c1171964c7e24c0f92f5019cf F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c b3d5ea919c2eef869306ec09e1b6e64a8b9fdd7421f93e6913f5c48b51b4ea6c +F src/btree.c 7e9673478702a5b26ebbbfc1e55a1f0b60bc147a52742db86f781e9fe96c6ecc F src/btree.h e8045998d865a21de226a417467cc64cf9d66e41b91b7436539f99ee464b5402 -F src/btreeInt.h 70236a051364c7d4b8c0513b118925a61cd24c2511ef3991e62732cfc8624c15 +F src/btreeInt.h 4a4cc19595e98c471c05cd9d7fc824da3397c303c7293019f4df7706caacf2fe F src/build.c 62faff0876fdce2b18f14312e3e891f3f366ee493c8fb48d844f9cbae15e6a02 F src/callback.c 8e14b60d1ed1c87c02cb5f121ecda99224f2aea6524a77ee6f72c9b5c7110f84 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e @@ -444,7 +444,7 @@ F src/os_unix.c 30e2c43e4955db990e5b5a81e901f8aa74cc8820 F src/os_win.c 2a6c73eef01c51a048cc4ddccd57f981afbec18a F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 7666599a49ff5098cfaea232cae118ad9afda37a96c9afa8f6db656a977c9cb6 -F src/pager.h ba7b3fbb18f78835a7368d66530bf21d6782e3e640d79c9550d218742c0b07ce +F src/pager.h 13713b181c041691a2bb7afbfbc4aa66c5addd34c46b1e68429eab75a5cfa632 F src/parse.y e4dbcc6136fc4f3084cedd41036401eafd336c5f7cb33bc516fed913a46cac2a F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 521bb9610d38ef17a3cc9b5ddafd4546c2ea67fa3d0e464823d73c2a28d50e11 @@ -1648,7 +1648,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 79544fc2856f30cac8b0962d00698974e8918562f09769a68264d17e1e1176fe a90c062d46c63a1e6f83064b1c5afb26a16e93b6ee8620ca46d169fdb325c488 -R 5ca93a16d61103201899491d16be4992 +P 213c61cb82d4ee51cc638fc5bb2b54cd51b963c40d088b2f01caf77032ea32fb +R d95fefd02901737ecfdecd42852575c7 U drh -Z 646329efaa2d2b6e9e011be806365053 +Z fe3c771992759cd0f7884a1e6c1e626c diff --git a/manifest.uuid b/manifest.uuid index 4fc907c6af..7b0dec5b1a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -213c61cb82d4ee51cc638fc5bb2b54cd51b963c40d088b2f01caf77032ea32fb \ No newline at end of file +a29401e924e6f82ca1c3eec98c6c0b2f1d5b9d7fba9af41a78516f21a7bab3a2 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index ca9e42af7b..711e20fdda 100644 --- a/src/btree.c +++ b/src/btree.c @@ -10314,6 +10314,9 @@ int sqlite3BtreeExclusiveLock(Btree *p){ assert( p->inTrans==TRANS_WRITE && pBt->pPage1 ); sqlite3BtreeEnter(p); rc = sqlite3PagerExclusiveLock(pBt->pPager, pBt->pPage1->pDbPage, &pgno); +#ifdef SQLITE_OMIT_CONCURRENT + assert( pgno==0 ); +#else if( rc==SQLITE_BUSY_SNAPSHOT && pgno ){ PgHdr *pPg = 0; int rc2 = sqlite3PagerGet(pBt->pPager, pgno, &pPg, 0); @@ -10362,7 +10365,7 @@ int sqlite3BtreeExclusiveLock(Btree *p){ ); } } - +#endif sqlite3BtreeLeave(p); return rc; } diff --git a/src/btreeInt.h b/src/btreeInt.h index fff805b962..44f1e015fb 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -277,6 +277,9 @@ struct MemPage { u8 intKey; /* True if table b-trees. False for index b-trees */ u8 intKeyLeaf; /* True if the leaf of an intKey table */ Pgno pgno; /* Page number for this page */ +#ifndef SQLITE_OMIT_CONCURRENT + Pgno pgnoRoot; /* Root page of b-tree that this page belongs to */ +#endif /* Only the first 8 bytes (above) are zeroed by pager.c when a new page ** is allocated. All fields that follow must be initialized before use */ u8 leaf; /* True if a leaf page */ @@ -301,9 +304,6 @@ struct MemPage { DbPage *pDbPage; /* Pager page handle */ u16 (*xCellSize)(MemPage*,u8*); /* cellSizePtr method */ void (*xParseCell)(MemPage*,u8*,CellInfo*); /* btreeParseCell method */ -#ifndef SQLITE_OMIT_CONCURRENT - u32 pgnoRoot; /* Root page of b-tree that this page belongs to */ -#endif }; /* diff --git a/src/pager.h b/src/pager.h index 8aa17c4553..7b02818b2f 100644 --- a/src/pager.h +++ b/src/pager.h @@ -229,7 +229,9 @@ int sqlite3PagerIsWal(Pager*); # define sqlite3PagerEndConcurrent(x) #endif +#if !defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_CONCURRENT) int sqlite3PagerIswriteable(DbPage*); +#endif int sqlite3PagerWalInfo(Pager*, u32 *pnPrior, u32 *pnFrame); From 1af0a4e1dfc2edfb560d18f410756d78ca6d1235 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Jul 2017 18:56:21 +0000 Subject: [PATCH 063/179] Fix a faulty #ifdef on the previous check-in FossilOrigin-Name: 7355e74239fc20f3d948867902675069e6a4efd103bf734f1b24a132f6d2edd1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.h | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 6ae55d542a..844cae323a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\sthe\sbuild\sworking\swith\sSQLITE_OMIT_CONCURRENT. -D 2017-07-20T18:28:33.979 +C Fix\sa\sfaulty\s#ifdef\son\sthe\sprevious\scheck-in +D 2017-07-20T18:56:21.537 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016 @@ -444,7 +444,7 @@ F src/os_unix.c 30e2c43e4955db990e5b5a81e901f8aa74cc8820 F src/os_win.c 2a6c73eef01c51a048cc4ddccd57f981afbec18a F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 7666599a49ff5098cfaea232cae118ad9afda37a96c9afa8f6db656a977c9cb6 -F src/pager.h 13713b181c041691a2bb7afbfbc4aa66c5addd34c46b1e68429eab75a5cfa632 +F src/pager.h 403b7d242616f8742ec149ede4a06b3e66896c554ce09c2abeddb926fdd1b707 F src/parse.y e4dbcc6136fc4f3084cedd41036401eafd336c5f7cb33bc516fed913a46cac2a F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 521bb9610d38ef17a3cc9b5ddafd4546c2ea67fa3d0e464823d73c2a28d50e11 @@ -1648,7 +1648,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 213c61cb82d4ee51cc638fc5bb2b54cd51b963c40d088b2f01caf77032ea32fb -R d95fefd02901737ecfdecd42852575c7 +P a29401e924e6f82ca1c3eec98c6c0b2f1d5b9d7fba9af41a78516f21a7bab3a2 +R 266b56d38ac33675de86b579a395b93a U drh -Z fe3c771992759cd0f7884a1e6c1e626c +Z 6866fd6a6d6e7ac87e940c57657cce02 diff --git a/manifest.uuid b/manifest.uuid index 7b0dec5b1a..d569dac3aa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a29401e924e6f82ca1c3eec98c6c0b2f1d5b9d7fba9af41a78516f21a7bab3a2 \ No newline at end of file +7355e74239fc20f3d948867902675069e6a4efd103bf734f1b24a132f6d2edd1 \ No newline at end of file diff --git a/src/pager.h b/src/pager.h index 7b02818b2f..70194e89aa 100644 --- a/src/pager.h +++ b/src/pager.h @@ -229,7 +229,7 @@ int sqlite3PagerIsWal(Pager*); # define sqlite3PagerEndConcurrent(x) #endif -#if !defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_CONCURRENT) +#if defined(SQLITE_DEBUG) || !defined(SQLITE_OMIT_CONCURRENT) int sqlite3PagerIswriteable(DbPage*); #endif From 0a2afca904f9ae11b44e17ea112cc07b2a78ef72 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Jul 2017 19:08:35 +0000 Subject: [PATCH 064/179] Fix compiler warnings. FossilOrigin-Name: 44c760d150b494ddd88382360cfcc4734fec936ed620f88b9df33ea7215c2fc2 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 1 - src/pager.c | 1 + src/wal.c | 7 ++++--- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 844cae323a..4757f0c129 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sfaulty\s#ifdef\son\sthe\sprevious\scheck-in -D 2017-07-20T18:56:21.537 +C Fix\scompiler\swarnings. +D 2017-07-20T19:08:35.935 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016 @@ -399,7 +399,7 @@ F src/auth.c 79f96c6f33bf0e5da8d1c282cee5ebb1852bb8a6ccca3e485d7c459b035d9c3c F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c c77b7f5759e413c1c8b53267d633c952e66db79c1171964c7e24c0f92f5019cf F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 7e9673478702a5b26ebbbfc1e55a1f0b60bc147a52742db86f781e9fe96c6ecc +F src/btree.c a21ffc82236d6108633559c9459d67b0e173e82934d417ad2ae998511ede6b55 F src/btree.h e8045998d865a21de226a417467cc64cf9d66e41b91b7436539f99ee464b5402 F src/btreeInt.h 4a4cc19595e98c471c05cd9d7fc824da3397c303c7293019f4df7706caacf2fe F src/build.c 62faff0876fdce2b18f14312e3e891f3f366ee493c8fb48d844f9cbae15e6a02 @@ -443,7 +443,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c 30e2c43e4955db990e5b5a81e901f8aa74cc8820 F src/os_win.c 2a6c73eef01c51a048cc4ddccd57f981afbec18a F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c 7666599a49ff5098cfaea232cae118ad9afda37a96c9afa8f6db656a977c9cb6 +F src/pager.c f29bab45ff5a2c51083196f4bb72d6ad1ab9d1811368f97e52b50579b6e19677 F src/pager.h 403b7d242616f8742ec149ede4a06b3e66896c554ce09c2abeddb926fdd1b707 F src/parse.y e4dbcc6136fc4f3084cedd41036401eafd336c5f7cb33bc516fed913a46cac2a F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 @@ -535,7 +535,7 @@ F src/vdbesort.c f512c68d0bf7e0105316a5594c4329358c8ee9cae3b25138df041d97516c037 F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834 F src/vtab.c 35b9bdc2b41de32a417141d12097bcc4e29a77ed7cdb8f836d1d2305d946b61b F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c e006353c3578071a457888aedd5eec9c8f1ba31ba11ddac4084c91c80478ca16 +F src/wal.c 57c8c0720420b5b4a4301b263f3cb09f68cf4cd2240f4473de0910b91de03923 F src/wal.h 1ea51dc499d6451529b822a8aaac053eafeef10b7fd9e5a4c9cc413182be429f F src/walker.c a7ca64ce08a83a20d32186fbe06bca9234e348cfcf07959ee322fdc3e8a6173a F src/where.c cbe8ddffbcec7ce86f7a800fe8fd10aee412c76c87e0dd3732a1682e68d74cd9 @@ -1648,7 +1648,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a29401e924e6f82ca1c3eec98c6c0b2f1d5b9d7fba9af41a78516f21a7bab3a2 -R 266b56d38ac33675de86b579a395b93a +P 7355e74239fc20f3d948867902675069e6a4efd103bf734f1b24a132f6d2edd1 +R 9b106d6151f13525ebfd4ea2a990e095 U drh -Z 6866fd6a6d6e7ac87e940c57657cce02 +Z 89e2d0fc2610ca8ab68df1aaa870de84 diff --git a/manifest.uuid b/manifest.uuid index d569dac3aa..a547a92a81 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7355e74239fc20f3d948867902675069e6a4efd103bf734f1b24a132f6d2edd1 \ No newline at end of file +44c760d150b494ddd88382360cfcc4734fec936ed620f88b9df33ea7215c2fc2 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 711e20fdda..4faaf31413 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4076,7 +4076,6 @@ static int btreeRelocateRange( rc = allocateBtreePage(pBt, &pFree, &iNew, iFirst-1, BTALLOC_LE); assert( rc!=SQLITE_OK || iNeweType, pEntry->parent,iNew,1); diff --git a/src/pager.c b/src/pager.c index 6b8e2ccddc..416ca997cb 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5474,6 +5474,7 @@ static int getPageNormal( /* If this is an CONCURRENT transaction and the page being read was ** present in the database file when the transaction was opened, ** mark it as read in the pAllRead vector. */ + pPg = 0; if( pPager->pAllRead && pgno<=pPager->dbOrigSize ){ rc = sqlite3BitvecSet(pPager->pAllRead, pgno); if( rc!=SQLITE_OK ) goto pager_acquire_err; diff --git a/src/wal.c b/src/wal.c index 13e1c57a99..25df6f3acf 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2899,9 +2899,10 @@ int sqlite3WalLockForCommit( rc = walHashGet(pWal, iHash, &aHash, &aPgno, &iZero); if( rc==SQLITE_OK ){ - int i; - int iMin = (iFirst - iZero); - int iMax = (iHash==0) ? HASHTABLE_NPAGE_ONE : HASHTABLE_NPAGE; + u32 i, iMin, iMax; + assert( iFirst >= iZero ); + iMin = (iFirst - iZero); + iMax = (iHash==0) ? HASHTABLE_NPAGE_ONE : HASHTABLE_NPAGE; if( iMin<1 ) iMin = 1; if( iMax>head.mxFrame ) iMax = head.mxFrame; for(i=iMin; rc==SQLITE_OK && i<=iMax; i++){ From c5856def1e192c086502f703dc197907497912f8 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Jul 2017 19:28:49 +0000 Subject: [PATCH 065/179] Minor fix to the concurrent.test module so that it works on windows. FossilOrigin-Name: 58553d61d199477f88ea9b45055ddaf821eca2aff1bf67c7d81fa80b23c2575a --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/concurrent.test | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 4757f0c129..a85bd5c4b6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompiler\swarnings. -D 2017-07-20T19:08:35.935 +C Minor\sfix\sto\sthe\sconcurrent.test\smodule\sso\sthat\sit\sworks\son\swindows. +D 2017-07-20T19:28:49.858 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016 @@ -659,7 +659,7 @@ F test/collateB.test 1e68906951b846570f29f20102ed91d29e634854ee47454d725f2151eca F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b F test/concfault.test 500f17c3fcfe7705114422bcc6ddd3c740001a43 -F test/concurrent.test 3eb5e6a911dc6ff72e3a679f563e683b436f6c701e6e1d6050173df2b8448d6b +F test/concurrent.test 75977c464f9613045dfa45eb9871aca0c51c2d67a6c89ccd19d47a88c5168cfe F test/concurrent2.test 9dfbeb0a323733fe1d13443371734bb94a674dbf777f464365475903873111f8 F test/concurrent3.test f4af1cf1220908c6dd5694923621c19e999b78cd997e2646285f08a52bcb4170 F test/concurrent4.test 653de3066911acfb9dcf3802bf4f1981b392b86c11f75e2c38ed1abfdd162293 @@ -1648,7 +1648,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7355e74239fc20f3d948867902675069e6a4efd103bf734f1b24a132f6d2edd1 -R 9b106d6151f13525ebfd4ea2a990e095 +P 44c760d150b494ddd88382360cfcc4734fec936ed620f88b9df33ea7215c2fc2 +R e43e51f268de0c519f41dfe98cdb868f U drh -Z 89e2d0fc2610ca8ab68df1aaa870de84 +Z 893b46e0693cb4cb333312e0b8e9a7df diff --git a/manifest.uuid b/manifest.uuid index a547a92a81..c116e6c3aa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -44c760d150b494ddd88382360cfcc4734fec936ed620f88b9df33ea7215c2fc2 \ No newline at end of file +58553d61d199477f88ea9b45055ddaf821eca2aff1bf67c7d81fa80b23c2575a \ No newline at end of file diff --git a/test/concurrent.test b/test/concurrent.test index 92a9879ba8..8b05668382 100644 --- a/test/concurrent.test +++ b/test/concurrent.test @@ -162,6 +162,8 @@ do_test 1.8.5 { db eval COMMIT db2 eval COMMIT } {} +db close +db2 close do_multiclient_test tn { @@ -557,4 +559,3 @@ do_multiclient_test tn { } finish_test - From 741802e825d69699e41e8635fb53dbda593d417f Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 11 Aug 2017 19:31:53 +0000 Subject: [PATCH 066/179] Add a PAGERTRACE() macro to log when frames are written into the WAL file. This is for analysis only and is a no-op for production builds. FossilOrigin-Name: 39f39e3d9a083eebe2f8dd5812d20c5b07cc17607b04b573a52fde6d00666f36 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 03713dce17..083bfe611d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\senhancements\sfrom\strunk. -D 2017-08-11T19:16:55.071 +C Add\sa\sPAGERTRACE()\smacro\sto\slog\swhen\sframes\sare\swritten\sinto\sthe\sWAL\sfile.\nThis\sis\sfor\sanalysis\sonly\sand\sis\sa\sno-op\sfor\sproduction\sbuilds. +D 2017-08-11T19:31:53.502 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016 @@ -444,7 +444,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c a361273749229755f92c8f0e3e4855054ad39bbc5c65773e8db5d0b79afa632c F src/os_win.c 964165b66cde03abc72fe948198b01be608436894732eadb94c8720d2467f223 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c 399d8d5f9436f2e14c5b116875248f03afacb05e77f8465b3f23ce7cad34c86d +F src/pager.c 7fbee7d787ada94b523cb50f92cd89b0eb9a297a1b4328260748b3df0619140e F src/pager.h 403b7d242616f8742ec149ede4a06b3e66896c554ce09c2abeddb926fdd1b707 F src/parse.y bcd8c5d9442518887df1fc9c35d454126927f705ae262cd3981c16bf297e910e F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 @@ -1657,7 +1657,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P aafe1fec803ddf207bbcb3c669da0a9639411375fc04a52066dde4eb99c3c11b 831156a4bd7c4408085f7c5584cdeebd1953c539972f80c5ef29bc147008630e -R cf26a5dc81c3e301583424514c738282 +P 47e716952d4a5d893b75726a2c52202cb0bc8ce1f75131e920a3ac2e202a507e +R dd67bd244e76571fe616a1fa57b01237 U drh -Z d7b2abd4937b3c1a53ce3f10f75e5d51 +Z d2720ec37e28e2f9ee94d83f86f8a988 diff --git a/manifest.uuid b/manifest.uuid index 6b3bea9342..99b0390e14 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -47e716952d4a5d893b75726a2c52202cb0bc8ce1f75131e920a3ac2e202a507e \ No newline at end of file +39f39e3d9a083eebe2f8dd5812d20c5b07cc17607b04b573a52fde6d00666f36 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 16838c7dc9..3b172d5e78 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3243,6 +3243,8 @@ static int pagerWalFrames( if( p->pgno<=nTruncate ){ ppNext = &p->pDirty; nList++; + PAGERTRACE(("TO-WAL %d page %d hash(%08x)\n", + PAGERID(pPager), p->pgno, pager_pagehash(p))); } } assert( pList ); From 9f16a96d4bc71ea54d61a3bbe706eb8350530b18 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 11 Aug 2017 20:22:30 +0000 Subject: [PATCH 067/179] Add another PAGERTRACE() macro to show when pages are added to Pager.pAllRead. No impact on production builds. FossilOrigin-Name: 11054cf5e8c24ef9ca869d558a0ca6750b56103c3b3793dc4afcef75192ea943 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 083bfe611d..10e8ee54df 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sPAGERTRACE()\smacro\sto\slog\swhen\sframes\sare\swritten\sinto\sthe\sWAL\sfile.\nThis\sis\sfor\sanalysis\sonly\sand\sis\sa\sno-op\sfor\sproduction\sbuilds. -D 2017-08-11T19:31:53.502 +C Add\sanother\sPAGERTRACE()\smacro\sto\sshow\swhen\spages\sare\sadded\sto\sPager.pAllRead.\nNo\simpact\son\sproduction\sbuilds. +D 2017-08-11T20:22:30.659 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016 @@ -444,7 +444,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c a361273749229755f92c8f0e3e4855054ad39bbc5c65773e8db5d0b79afa632c F src/os_win.c 964165b66cde03abc72fe948198b01be608436894732eadb94c8720d2467f223 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c 7fbee7d787ada94b523cb50f92cd89b0eb9a297a1b4328260748b3df0619140e +F src/pager.c fac516ae6a04a45b2e84804245d0d0c38be07e8dd62be1f7613c83913f681570 F src/pager.h 403b7d242616f8742ec149ede4a06b3e66896c554ce09c2abeddb926fdd1b707 F src/parse.y bcd8c5d9442518887df1fc9c35d454126927f705ae262cd3981c16bf297e910e F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 @@ -1657,7 +1657,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 47e716952d4a5d893b75726a2c52202cb0bc8ce1f75131e920a3ac2e202a507e -R dd67bd244e76571fe616a1fa57b01237 +P 39f39e3d9a083eebe2f8dd5812d20c5b07cc17607b04b573a52fde6d00666f36 +R 44b0453e47b7d15fed17613bc4cf70e5 U drh -Z d2720ec37e28e2f9ee94d83f86f8a988 +Z f6a60fb915e7da3c7036aeb9a03bed52 diff --git a/manifest.uuid b/manifest.uuid index 99b0390e14..e9bedf7cbe 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -39f39e3d9a083eebe2f8dd5812d20c5b07cc17607b04b573a52fde6d00666f36 \ No newline at end of file +11054cf5e8c24ef9ca869d558a0ca6750b56103c3b3793dc4afcef75192ea943 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 3b172d5e78..733d65d436 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5500,6 +5500,7 @@ static int getPageNormal( ** mark it as read in the pAllRead vector. */ pPg = 0; if( pPager->pAllRead && pgno<=pPager->dbOrigSize ){ + PAGERTRACE(("USING page %d\n", pgno)); rc = sqlite3BitvecSet(pPager->pAllRead, pgno); if( rc!=SQLITE_OK ) goto pager_acquire_err; } From 6aab028db0695dbaa085d7334ac5cc59595a172f Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 11 Aug 2017 21:16:23 +0000 Subject: [PATCH 068/179] Fix a problem allowing a conflicting transaction to be committed in the case where more than one 32KB shared-memory page has been written to since the transaction was started. FossilOrigin-Name: 38dd9b50fe260d853cbc2551bc1bb60ddf5752f0456e0da3afe4cbf728c891d8 --- manifest | 16 ++++++------ manifest.uuid | 2 +- src/wal.c | 7 +++--- test/concurrent.test | 59 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 10e8ee54df..8893e26ec0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sanother\sPAGERTRACE()\smacro\sto\sshow\swhen\spages\sare\sadded\sto\sPager.pAllRead.\nNo\simpact\son\sproduction\sbuilds. -D 2017-08-11T20:22:30.659 +C Fix\sa\sproblem\sallowing\sa\sconflicting\stransaction\sto\sbe\scommitted\sin\sthe\scase\nwhere\smore\sthan\sone\s32KB\sshared-memory\spage\shas\sbeen\swritten\sto\ssince\sthe\ntransaction\swas\sstarted. +D 2017-08-11T21:16:23.047 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016 @@ -536,7 +536,7 @@ F src/vdbesort.c fea2bea25f5e9ccd91e0760d7359f0365f9fba1aaeac7216c71cad78765f58e F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834 F src/vtab.c f1d5c23132fb0247af3e86146404112283ddedb6c518de0d4edc91cfb36970ef F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 57c8c0720420b5b4a4301b263f3cb09f68cf4cd2240f4473de0910b91de03923 +F src/wal.c ab94c3cc7bfe006c424b5f5945ad8285ffcda36222f0d40048d46bafb43dab69 F src/wal.h 1ea51dc499d6451529b822a8aaac053eafeef10b7fd9e5a4c9cc413182be429f F src/walker.c a7ca64ce08a83a20d32186fbe06bca9234e348cfcf07959ee322fdc3e8a6173a F src/where.c cbe8ddffbcec7ce86f7a800fe8fd10aee412c76c87e0dd3732a1682e68d74cd9 @@ -662,7 +662,7 @@ F test/collateB.test 1e68906951b846570f29f20102ed91d29e634854ee47454d725f2151eca F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 F test/colname.test b111edd2a84f558567320904bb94c779d7eec47254265b5f0a3d1f3e52cc28e0 F test/concfault.test 500f17c3fcfe7705114422bcc6ddd3c740001a43 -F test/concurrent.test 75977c464f9613045dfa45eb9871aca0c51c2d67a6c89ccd19d47a88c5168cfe +F test/concurrent.test f91ab1933beee68b5dbe7a784ed147d2ad190c1b84425e3ea84aefe14f7d1151 F test/concurrent2.test 9dfbeb0a323733fe1d13443371734bb94a674dbf777f464365475903873111f8 F test/concurrent3.test f4af1cf1220908c6dd5694923621c19e999b78cd997e2646285f08a52bcb4170 F test/concurrent4.test 653de3066911acfb9dcf3802bf4f1981b392b86c11f75e2c38ed1abfdd162293 @@ -1657,7 +1657,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 39f39e3d9a083eebe2f8dd5812d20c5b07cc17607b04b573a52fde6d00666f36 -R 44b0453e47b7d15fed17613bc4cf70e5 -U drh -Z f6a60fb915e7da3c7036aeb9a03bed52 +P 11054cf5e8c24ef9ca869d558a0ca6750b56103c3b3793dc4afcef75192ea943 +R 4e06f85f8b10dfd7cc0854935df8167f +U dan +Z d88ae17a00fc8079da60ecbd46144cbb diff --git a/manifest.uuid b/manifest.uuid index e9bedf7cbe..f2738523b9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -11054cf5e8c24ef9ca869d558a0ca6750b56103c3b3793dc4afcef75192ea943 \ No newline at end of file +38dd9b50fe260d853cbc2551bc1bb60ddf5752f0456e0da3afe4cbf728c891d8 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 25df6f3acf..1e0b6f98bc 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2900,11 +2900,10 @@ int sqlite3WalLockForCommit( rc = walHashGet(pWal, iHash, &aHash, &aPgno, &iZero); if( rc==SQLITE_OK ){ u32 i, iMin, iMax; - assert( iFirst >= iZero ); - iMin = (iFirst - iZero); + assert( head.mxFrame>=iZero ); + iMin = (iZero >= iFirst) ? 1 : (iFirst - iZero); iMax = (iHash==0) ? HASHTABLE_NPAGE_ONE : HASHTABLE_NPAGE; - if( iMin<1 ) iMin = 1; - if( iMax>head.mxFrame ) iMax = head.mxFrame; + if( iMax>(head.mxFrame-iZero) ) iMax = (head.mxFrame-iZero); for(i=iMin; rc==SQLITE_OK && i<=iMax; i++){ PgHdr *pPg; if( aPgno[i]==1 ){ diff --git a/test/concurrent.test b/test/concurrent.test index 8b05668382..f18d9d8922 100644 --- a/test/concurrent.test +++ b/test/concurrent.test @@ -558,4 +558,63 @@ do_multiclient_test tn { } {1 {database is locked}} } +do_multiclient_test tn { + do_test 7.$tn.1 { + sql1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100) + INSERT INTO t1 SELECT NULL, randomblob(400) FROM s; + + CREATE TABLE t2(a INTEGER PRIMARY KEY, b); + WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<50000) + INSERT INTO t2 SELECT NULL, randomblob(400) FROM s; + + CREATE TABLE t3(a INTEGER PRIMARY KEY, b); + WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100) + INSERT INTO t3 SELECT NULL, randomblob(400) FROM s; + + CREATE TABLE t4(a INTEGER PRIMARY KEY, b); + + PRAGMA wal_checkpoint; + } + set {} {} + } {} + + do_test 7.$tn.2 { + sql2 { + BEGIN CONCURRENT; + SELECT * FROM t1; + INSERT INTO t4 VALUES(1, 2); + } + set {} {} + } {} + + do_test 7.$tn.3 { + sql3 { + BEGIN CONCURRENT; + SELECT * FROM t3; + INSERT INTO t4 VALUES(1, 2); + } + set {} {} + } {} + + do_test 7.$tn.4 { + sql1 { + UPDATE t1 SET b=randomblob(400); + UPDATE t2 SET b=randomblob(400); + UPDATE t3 SET b=randomblob(400); + } + } {} + + do_test 7.$tn.5 { + csql2 { COMMIT } + } {1 {database is locked}} + + do_test 7.$tn.6 { + csql3 { COMMIT } + } {1 {database is locked}} + +} + finish_test From d828d0debc064b64afa563c499d2d7b7e3b6f006 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 12 Aug 2017 14:06:15 +0000 Subject: [PATCH 069/179] Add a more rigorous test case for the bug fixed by the previous commit on this branch. FossilOrigin-Name: 4256072399f44f48ed0856aa8112226af6feaf8676923612bde6cea239ebf920 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/concurrent.test | 32 +++++++++++++++++++++++++++++++- 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 8893e26ec0..91cd200ac0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sallowing\sa\sconflicting\stransaction\sto\sbe\scommitted\sin\sthe\scase\nwhere\smore\sthan\sone\s32KB\sshared-memory\spage\shas\sbeen\swritten\sto\ssince\sthe\ntransaction\swas\sstarted. -D 2017-08-11T21:16:23.047 +C Add\sa\smore\srigorous\stest\scase\sfor\sthe\sbug\sfixed\sby\sthe\sprevious\scommit\son\sthis\nbranch. +D 2017-08-12T14:06:15.050 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016 @@ -662,7 +662,7 @@ F test/collateB.test 1e68906951b846570f29f20102ed91d29e634854ee47454d725f2151eca F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 F test/colname.test b111edd2a84f558567320904bb94c779d7eec47254265b5f0a3d1f3e52cc28e0 F test/concfault.test 500f17c3fcfe7705114422bcc6ddd3c740001a43 -F test/concurrent.test f91ab1933beee68b5dbe7a784ed147d2ad190c1b84425e3ea84aefe14f7d1151 +F test/concurrent.test a801cd60c370f0ed851657c9576b102f9ab1dd846c6a88e6ae45939a8deeda7c F test/concurrent2.test 9dfbeb0a323733fe1d13443371734bb94a674dbf777f464365475903873111f8 F test/concurrent3.test f4af1cf1220908c6dd5694923621c19e999b78cd997e2646285f08a52bcb4170 F test/concurrent4.test 653de3066911acfb9dcf3802bf4f1981b392b86c11f75e2c38ed1abfdd162293 @@ -1657,7 +1657,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 11054cf5e8c24ef9ca869d558a0ca6750b56103c3b3793dc4afcef75192ea943 -R 4e06f85f8b10dfd7cc0854935df8167f +P 38dd9b50fe260d853cbc2551bc1bb60ddf5752f0456e0da3afe4cbf728c891d8 +R 5ae6eec944d8695060377eb5fcb02f25 U dan -Z d88ae17a00fc8079da60ecbd46144cbb +Z be5258dc78cb84266e8cf9b3ea9ebd92 diff --git a/manifest.uuid b/manifest.uuid index f2738523b9..d2eec08d49 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -38dd9b50fe260d853cbc2551bc1bb60ddf5752f0456e0da3afe4cbf728c891d8 \ No newline at end of file +4256072399f44f48ed0856aa8112226af6feaf8676923612bde6cea239ebf920 \ No newline at end of file diff --git a/test/concurrent.test b/test/concurrent.test index f18d9d8922..b8f8d7e0d6 100644 --- a/test/concurrent.test +++ b/test/concurrent.test @@ -576,7 +576,6 @@ do_multiclient_test tn { CREATE TABLE t4(a INTEGER PRIMARY KEY, b); - PRAGMA wal_checkpoint; } set {} {} } {} @@ -615,6 +614,37 @@ do_multiclient_test tn { csql3 { COMMIT } } {1 {database is locked}} + + csql2 ROLLBACK + csql3 ROLLBACK + + # The following test works with $tn==1 (sql2 and sql3 use separate + # processes), but is quite slow. So only run it with $tn==2 (all + # connections in the same process). + # + if {$tn==2} { + do_test 7.$tn.7 { + for {set i 1} {$i < 10000} {incr i} { + sql3 { + PRAGMA wal_checkpoint; + BEGIN CONCURRENT; + SELECT * FROM t3; + INSERT INTO t4 VALUES(1, 2); + } + + sql1 { + UPDATE t2 SET b = randomblob(400) WHERE rowid <= $i; + UPDATE t3 SET b = randomblob(400) WHERE rowid = 1; + } + + if {[csql3 COMMIT]!={1 {database is locked}}} { + error "Failed at i=$i" + } + csql3 ROLLBACK + } + } {} + } + } finish_test From 9c54156a9cc203a106f9938eab1a572c7f3598f3 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 24 Aug 2017 10:10:28 +0000 Subject: [PATCH 070/179] Test BEGIN CONCURRENT transactions that consist entirely of read-only statements. FossilOrigin-Name: c3fe1f4b7e8dcadcb516622719d000b808effe3ad497244ba44f57d52dc2cc08 --- manifest | 12 +++--- manifest.uuid | 2 +- test/concurrent4.test | 89 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 91cd200ac0..c5496f5966 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\smore\srigorous\stest\scase\sfor\sthe\sbug\sfixed\sby\sthe\sprevious\scommit\son\sthis\nbranch. -D 2017-08-12T14:06:15.050 +C Test\sBEGIN\sCONCURRENT\stransactions\sthat\sconsist\sentirely\sof\sread-only\nstatements. +D 2017-08-24T10:10:28.736 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016 @@ -665,7 +665,7 @@ F test/concfault.test 500f17c3fcfe7705114422bcc6ddd3c740001a43 F test/concurrent.test a801cd60c370f0ed851657c9576b102f9ab1dd846c6a88e6ae45939a8deeda7c F test/concurrent2.test 9dfbeb0a323733fe1d13443371734bb94a674dbf777f464365475903873111f8 F test/concurrent3.test f4af1cf1220908c6dd5694923621c19e999b78cd997e2646285f08a52bcb4170 -F test/concurrent4.test 653de3066911acfb9dcf3802bf4f1981b392b86c11f75e2c38ed1abfdd162293 +F test/concurrent4.test e0b12cd467137e50259df3b4f837507e82aaa07c35941c88664dc8ed1d089c44 F test/concurrent5.test d5d7d9d404a9b4502464fc097c1fc5c3012bb4f1b063fae7ad707ca983fc86c5 F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c @@ -1657,7 +1657,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 38dd9b50fe260d853cbc2551bc1bb60ddf5752f0456e0da3afe4cbf728c891d8 -R 5ae6eec944d8695060377eb5fcb02f25 +P 4256072399f44f48ed0856aa8112226af6feaf8676923612bde6cea239ebf920 +R 03a159c270fd82b6d54f5498b523d32e U dan -Z be5258dc78cb84266e8cf9b3ea9ebd92 +Z fcb6179847dc0180b01d1fb922b3b8b6 diff --git a/manifest.uuid b/manifest.uuid index d2eec08d49..7d481eeeee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4256072399f44f48ed0856aa8112226af6feaf8676923612bde6cea239ebf920 \ No newline at end of file +c3fe1f4b7e8dcadcb516622719d000b808effe3ad497244ba44f57d52dc2cc08 \ No newline at end of file diff --git a/test/concurrent4.test b/test/concurrent4.test index dd29434381..d43e701d99 100644 --- a/test/concurrent4.test +++ b/test/concurrent4.test @@ -101,5 +101,94 @@ do_execsql_test 3.2 { COMMIT; } {} +#------------------------------------------------------------------------- +# Test the effect of BEGIN CONCURRENT transactions that consist entirely +# of read-only statements. +# +reset_db +do_execsql_test 4.0 { + PRAGMA page_size = 1024; + PRAGMA journal_mode = wal; + + CREATE TABLE t4(a, b); + INSERT INTO t4 VALUES(1, 2); + INSERT INTO t4 VALUES(3, 4); +} {wal} + +sqlite3 db2 test.db +do_test 4.1.1 { + db eval { + BEGIN CONCURRENT; + INSERT INTO t4 VALUES(5, 6); + } + + db2 eval { + BEGIN CONCURRENT; + SELECT * FROM t4; + ROLLBACK; + } +} {1 2 3 4} + +do_test 4.1.2 { + db eval { COMMIT } + db2 eval { SELECT * FROM t4 } +} {1 2 3 4 5 6} + +do_test 4.2.1 { + db eval { + BEGIN CONCURRENT; + INSERT INTO t4 VALUES(7, 8); + } + + db2 eval { + BEGIN CONCURRENT; + SELECT * FROM t4; + COMMIT; + } +} {1 2 3 4 5 6} + +do_test 4.2.2 { + db eval { COMMIT } + db2 eval { SELECT * FROM t4 } +} {1 2 3 4 5 6 7 8} + +do_test 4.3 { + db2 eval { + BEGIN CONCURRENT; + SELECT * FROM t4; + } + + db eval { + BEGIN CONCURRENT; + INSERT INTO t4 VALUES(9, 10); + COMMIT; + } + db2 eval { + SELECT * FROM t4; + COMMIT; + } +} {1 2 3 4 5 6 7 8} + +set sz [file size test.db-wal] +do_test 4.4.1 { + db eval { + BEGIN CONCURRENT; + SELECT * FROM t4; + SELECT * FROM sqlite_master; + } + + db eval COMMIT + file size test.db-wal +} $sz +do_test 4.4.2 { + db eval { + BEGIN CONCURRENT; + SELECT * FROM t4; + SELECT * FROM sqlite_master; + ROLLBACK; + } + file size test.db-wal +} $sz + finish_test From 03d405f0464f25de0ef1aa76a650faa1baabf931 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 12 Sep 2017 20:09:31 +0000 Subject: [PATCH 071/179] Add the highly-experimental "PRAGMA noop_update=TRUE" command. FossilOrigin-Name: afe45271b9c0cd379cf0beb94657e2396068c4a18f84003c4c48297760fd83ee --- manifest | 24 ++++++++++++++---------- manifest.uuid | 2 +- src/pragma.h | 11 ++++++++++- src/sqliteInt.h | 1 + src/update.c | 11 +++++++++++ tool/mkpragmatab.tcl | 6 ++++++ 6 files changed, 43 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 91cd200ac0..aba94857ed 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\smore\srigorous\stest\scase\sfor\sthe\sbug\sfixed\sby\sthe\sprevious\scommit\son\sthis\nbranch. -D 2017-08-12T14:06:15.050 +C Add\sthe\shighly-experimental\s"PRAGMA\snoop_update=TRUE"\scommand. +D 2017-09-12T20:09:31.560 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016 @@ -451,7 +451,7 @@ F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 521bb9610d38ef17a3cc9b5ddafd4546c2ea67fa3d0e464823d73c2a28d50e11 F src/pcache1.c 0b793738b5dddaf0a645784835c6b5557b1ecfaee339af9c26810c6ecdb273aa F src/pragma.c cd6aeda3587be6c5c08f9b2d45eae6068666a03c9d077c8c43cdb85fb0aa70f2 -F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324 +F src/pragma.h 70cb22e66adabf21bac20e2894184ece986a67a3252506220f3b62633c87dceb F src/prepare.c 3cbb99757d7295997674972f9dd2331c5c544368854ca08954c9beb1e9b6145a F src/printf.c 8757834f1b54dae512fb25eb1acc8e94a0d15dd2290b58f2563f65973265adb2 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 @@ -463,7 +463,7 @@ F src/shell.c.in b5725acacba95ccefa57b6d068f710e29ba8239c3aa704628a1902a1f729c17 F src/sqlite.h.in 803d2c969bccaf78ef087269c73f1f00f8870c122b3514414b8c47c4fde73e82 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47 -F src/sqliteInt.h 0eccf839f75dab42f46559ea529c51921b8bf427b3d9a170a45e8e58cb4304bd +F src/sqliteInt.h dc8a03250ca4887630a048ed135e50c47f75ef30a0859256420d072e6b98caf4 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -521,7 +521,7 @@ F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 1003d6d90c6783206c711f0a9397656fa5b055209f4d092caa43bb3bf5215db5 F src/treeview.c 2ee4a5dada213d5ab08a742af5c876cee6f1aaae65f10a61923f3fb63846afef F src/trigger.c 48e0f7ed6749ce4d50a695e09e20ce9cf84ecabf2691852c965a51e0b620eccc -F src/update.c 5404be9e840717323a69209190cdbc9d0d34adaedaaf1d1a1069babf2c4171c0 +F src/update.c 2bb1d048a1a56e1f9238d68e8e476299a5873c4d4d9e0e9c4ab542d8b25f341d F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 F src/util.c fc081ec6f63448dcd80d3dfad35baecfa104823254a815b081a4d9fe76e1db23 F src/vacuum.c 07eec96f821c7dcbdca2fadffc6e38ea2c24bf409fcb15fe9fb3ac444d632dfe @@ -1594,7 +1594,7 @@ F tool/mkmsvcmin.tcl cbd93f1cfa3a0a9ae56fc958510aa3fc3ac65e29cb111716199e3d0e66e F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c F tool/mkopcodeh.tcl 4ee2a30ccbd900dc4d5cdb61bdab87cd2166cd2affcc78c9cc0b8d22a65b2eee F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e -F tool/mkpragmatab.tcl 2144bc8550a6471a029db262a132d2df4b9e0db61b90398bf64f5b7b3f8d92cd +F tool/mkpragmatab.tcl e9a725395dc4419047d242f4ebd4716dd8c682e234b6553a4bb891fc1f12174c F tool/mkshellc.tcl 69c38ecd7b74b2b0799a35ce20e1e3998e504d8c99c100ca4b98ae9d8f6279bc F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb @@ -1657,7 +1657,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 38dd9b50fe260d853cbc2551bc1bb60ddf5752f0456e0da3afe4cbf728c891d8 -R 5ae6eec944d8695060377eb5fcb02f25 -U dan -Z be5258dc78cb84266e8cf9b3ea9ebd92 +P 4256072399f44f48ed0856aa8112226af6feaf8676923612bde6cea239ebf920 +Q +de2e371757d2031cefc4bbae29d746a768c126a6c8443bb0f9ddebee7e69240b +R 2413782f3f576a461bb90e166b6f6e32 +T *branch * begin-concurrent-pnu +T *sym-begin-concurrent-pnu * +T -sym-begin-concurrent * +U drh +Z 105db701de986bc4ba15a35d9febe596 diff --git a/manifest.uuid b/manifest.uuid index d2eec08d49..c3c05f6cd6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4256072399f44f48ed0856aa8112226af6feaf8676923612bde6cea239ebf920 \ No newline at end of file +afe45271b9c0cd379cf0beb94657e2396068c4a18f84003c4c48297760fd83ee \ No newline at end of file diff --git a/src/pragma.h b/src/pragma.h index c9ece2dc87..80801945cf 100644 --- a/src/pragma.h +++ b/src/pragma.h @@ -440,6 +440,15 @@ static const PragmaName aPragmaName[] = { /* iArg: */ 0 }, #endif #endif +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if defined(SQLITE_DEBUG) + {/* zName: */ "noop_update", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_NoopUpdate }, +#endif #endif {/* zName: */ "optimize", /* ePragTyp: */ PragTyp_OPTIMIZE, @@ -646,4 +655,4 @@ static const PragmaName aPragmaName[] = { /* iArg: */ SQLITE_WriteSchema }, #endif }; -/* Number of pragmas: 60 on by default, 77 total. */ +/* Number of pragmas: 60 on by default, 78 total. */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 7d41119578..c6295c3d33 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1490,6 +1490,7 @@ struct sqlite3 { #define SQLITE_EnableQPSG 0x00800000 /* Query Planner Stability Guarantee */ /* Flags used only if debugging */ #ifdef SQLITE_DEBUG +#define SQLITE_NoopUpdate 0x04000000 /* UPDATE operations are no-ops */ #define SQLITE_SqlTrace 0x08000000 /* Debug print SQL as it executes */ #define SQLITE_VdbeListing 0x10000000 /* Debug listings of VDBE programs */ #define SQLITE_VdbeTrace 0x20000000 /* True to trace VDBE execution */ diff --git a/src/update.c b/src/update.c index e69efdb6bf..912b7e646b 100644 --- a/src/update.c +++ b/src/update.c @@ -223,6 +223,17 @@ void sqlite3Update( */ chngRowid = chngPk = 0; for(i=0; inExpr; i++){ +#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_FLAG_PRAGMAS) + if( db->flags & SQLITE_NoopUpdate ){ + Token x; + sqlite3ExprDelete(db, pChanges->a[i].pExpr); + x.z = pChanges->a[i].zName; + x.n = sqlite3Strlen30(x.z); + pChanges->a[i].pExpr = + sqlite3PExpr(pParse, TK_UPLUS, sqlite3ExprAlloc(db, TK_ID, &x, 0), 0); + if( db->mallocFailed ) goto update_cleanup; + } +#endif if( sqlite3ResolveExprNames(&sNC, pChanges->a[i].pExpr) ){ goto update_cleanup; } diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index 1bfbeb7c58..b790398476 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -112,6 +112,12 @@ set pragma_def { IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) IF: defined(SQLITE_DEBUG) + NAME: noop_update + TYPE: FLAG + ARG: SQLITE_NoopUpdate + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) + IF: defined(SQLITE_DEBUG) + NAME: ignore_check_constraints TYPE: FLAG ARG: SQLITE_IgnoreChecks From 7393c7424e14ab4acdc188a93b84d5bfaefa954b Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 22 Sep 2017 11:09:09 +0000 Subject: [PATCH 072/179] Cherrypick [ec37ad6d08] into this branch. With this patch, if SQLITE_SHARED_MAPPING is defined at build-time SQLite will use a single memory mapping for multiple connections to the same database file within a single process. FossilOrigin-Name: c7a5880d6d898299b4c9414b7702cfa450aa5f7bf4ec8f417b94d2a7b6558264 --- manifest | 13 +++++++------ manifest.uuid | 2 +- src/os_unix.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index b3429762a7..861009c901 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\sinto\sthis\sbranch. -D 2017-09-22T10:49:03.297 +C Cherrypick\s[ec37ad6d08]\sinto\sthis\sbranch.\sWith\sthis\spatch,\sif\nSQLITE_SHARED_MAPPING\sis\sdefined\sat\sbuild-time\sSQLite\swill\suse\sa\ssingle\smemory\nmapping\sfor\smultiple\sconnections\sto\sthe\ssame\sdatabase\sfile\swithin\sa\ssingle\nprocess. +D 2017-09-22T11:09:09.656 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f @@ -446,7 +446,7 @@ F src/os.c 93e0979b9b55df29c0c4923f73b48e9d3fe728f01dd8ed4f6a9d2f1d79779bc8 F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c 3984fc069df59e26f000e30609611cecdb4e93293e6ee52313a473a7e874af1b +F src/os_unix.c 1b2203f30ec4ecef66b0f9b714432edaffe0f89f189af7c6b1db3e2cb3a52d24 F src/os_win.c 5c802f05e706c87c6e4cc6e9527f3364c7a7178458f93dffa5e19ac2e8eef9c1 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 84e90e0acee15766840f89b8be6f14322de16396d9229c494fec6c1cc03a69cf @@ -1666,7 +1666,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d53108e76307fb24ef1d8f75fe82c180f5588ad8e65eeceace92f4cecadfa452 0a12915b373cd0491a58d8f7a645711c620c70efced623e6b40aa01f23284157 -R 5b090dbbf85dc5792098185268ac7c8f +P 307b802e8627c93a51e4c54851a4fab33db5061bb80e3d327ce53b127d6d511b +Q +ec37ad6d08362f4c9faad9b629c0fa23f5864ff6ad7f4cbed93a25d5f7b815d8 +R a7069c27bdffb405c8574ad991612563 U dan -Z 51723e9e7fe6c543a37219e2d01ad740 +Z 9a1235c2d010c6dd3774c97b07deb6f2 diff --git a/manifest.uuid b/manifest.uuid index f7b69a87af..da8f0bb88e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -307b802e8627c93a51e4c54851a4fab33db5061bb80e3d327ce53b127d6d511b \ No newline at end of file +c7a5880d6d898299b4c9414b7702cfa450aa5f7bf4ec8f417b94d2a7b6558264 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 4445104dd6..0e444e61f2 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -1115,6 +1115,10 @@ struct unixInodeInfo { sem_t *pSem; /* Named POSIX semaphore */ char aSemName[MAX_PATHNAME+2]; /* Name of that semaphore */ #endif +#ifdef SQLITE_SHARED_MAPPING + sqlite3_int64 nSharedMapping; /* Size of mapped region in bytes */ + void *pSharedMapping; /* Memory mapped region */ +#endif }; /* @@ -1249,6 +1253,13 @@ static void releaseInodeInfo(unixFile *pFile){ pInode->nRef--; if( pInode->nRef==0 ){ assert( pInode->pShmNode==0 ); +#ifdef SQLITE_SHARED_MAPPING + if( pInode->pSharedMapping ){ + osMunmap(pInode->pSharedMapping, pInode->nSharedMapping); + pInode->pSharedMapping = 0; + pInode->nSharedMapping = 0; + } +#endif closePendingFds(pFile); if( pInode->pPrev ){ assert( pInode->pPrev->pNext==pInode ); @@ -2055,6 +2066,14 @@ static int nolockUnlock(sqlite3_file *NotUsed, int NotUsed2){ ** Close the file. */ static int nolockClose(sqlite3_file *id) { +#ifdef SQLITE_SHARED_MAPPING + unixFile *pFd = (unixFile*)id; + if( pFd->pInode ){ + unixEnterMutex(); + releaseInodeInfo(pFd); + unixLeaveMutex(); + } +#endif return closeUnixFile(id); } @@ -3874,6 +3893,9 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ *(i64*)pArg = pFile->mmapSizeMax; if( newLimit>=0 && newLimit!=pFile->mmapSizeMax && pFile->nFetchOut==0 ){ pFile->mmapSizeMax = newLimit; +#ifdef SQLITE_SHARED_MAPPING + if( pFile->pInode==0 ) +#endif if( pFile->mmapSize>0 ){ unixUnmapfile(pFile); rc = unixMapfile(pFile, -1); @@ -4774,6 +4796,9 @@ static int unixShmUnmap( */ static void unixUnmapfile(unixFile *pFd){ assert( pFd->nFetchOut==0 ); +#ifdef SQLITE_SHARED_MAPPING + if( pFd->pInode ) return; +#endif if( pFd->pMapRegion ){ osMunmap(pFd->pMapRegion, pFd->mmapSizeActual); pFd->pMapRegion = 0; @@ -4905,6 +4930,28 @@ static int unixMapfile(unixFile *pFd, i64 nMap){ nMap = pFd->mmapSizeMax; } +#ifdef SQLITE_SHARED_MAPPING + if( pFd->pInode ){ + unixInodeInfo *pInode = pFd->pInode; + if( pFd->pMapRegion ) return SQLITE_OK; + unixEnterMutex(); + if( pInode->pSharedMapping==0 ){ + u8 *pNew = osMmap(0, nMap, PROT_READ, MAP_SHARED, pFd->h, 0); + if( pNew==MAP_FAILED ){ + unixLogError(SQLITE_OK, "mmap", pFd->zPath); + pFd->mmapSizeMax = 0; + }else{ + pInode->pSharedMapping = pNew; + pInode->nSharedMapping = nMap; + } + } + pFd->pMapRegion = pInode->pSharedMapping; + pFd->mmapSizeActual = pFd->mmapSize = pInode->nSharedMapping; + unixLeaveMutex(); + return SQLITE_OK; + } +#endif + assert( nMap>0 || (pFd->mmapSize==0 && pFd->pMapRegion==0) ); if( nMap!=pFd->mmapSize ){ unixRemapfile(pFd, nMap); @@ -5342,6 +5389,9 @@ static int fillInUnixFile( if( pLockingStyle == &posixIoMethods #if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE || pLockingStyle == &nfsIoMethods +#endif +#ifdef SQLITE_SHARED_MAPPING + || pLockingStyle == &nolockIoMethods #endif ){ unixEnterMutex(); From 055cc1e431811d6e87eea672acd2e96e95138e07 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 4 Oct 2017 20:57:14 +0000 Subject: [PATCH 073/179] Add experimental mode that uses two wal files. Activated using "PRAGMA journal_mode = wal2". FossilOrigin-Name: e2fc5c814cf6862d536aacb9eca66ecd31ba7e3e3033fa4c5564d533f4a18dfc --- manifest | 41 +- manifest.uuid | 2 +- src/btree.c | 10 +- src/os_unix.c | 4 +- src/pager.c | 47 +- src/pager.h | 19 +- src/pragma.c | 3 +- src/vdbe.c | 62 +- src/wal.c | 1567 +++++++++++++++++++++++++++------------- src/wal.h | 4 +- test/permutations.test | 17 + test/savepoint.test | 1 + test/savepoint6.test | 17 +- test/tester.tcl | 24 +- test/waltwo2.test | 127 ++++ 15 files changed, 1363 insertions(+), 582 deletions(-) create mode 100644 test/waltwo2.test diff --git a/manifest b/manifest index 1ea0bfb8d7..f6dbc867fd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sto\srequirements\smarks. -D 2017-10-04T18:26:44.610 +C Add\sexperimental\smode\sthat\suses\stwo\swal\sfiles.\sActivated\susing\s"PRAGMA\njournal_mode\s=\swal2". +D 2017-10-04T20:57:14.949 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f @@ -401,7 +401,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c cc88a7fca7287dfc004921bb5e2764893dfe4f6dd33be3570126b3fc37932600 +F src/btree.c c7d94975b015058e6a68278e53d841585552bbc541c1b31b6e024eb9fd888746 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc F src/build.c e71e96a67daf3d1dd23188423e66cd6af38017e2ec73fead5d2b57da2d3c7e16 @@ -442,16 +442,16 @@ F src/os.c 93e0979b9b55df29c0c4923f73b48e9d3fe728f01dd8ed4f6a9d2f1d79779bc8 F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c 3984fc069df59e26f000e30609611cecdb4e93293e6ee52313a473a7e874af1b +F src/os_unix.c 1f9c3e771557edd248e1fcec0818739826b2a3121b609a6e3372b670236d085a F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c 2aa56a99bb13128d9102e84c7a9f835e546cbb58f0861d481bc3db32973b1628 -F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a +F src/pager.c bc2d2b6784cc2b5103e07aff13b012dab30c7efa0abb9bc2bb949ec45cba4326 +F src/pager.h e11e516208a460bea1b95fe3da697642306e7f350d5f96d2c1d21231ee4d2bf2 F src/parse.y 52ef3cecd0934e9da4a45b585883a03243ad615d338ad94f44501a05891dcdfa F src/pcache.c 4bada070456980c3c1f16d58ec2e64e389ad77b935e3d77e0c96e7bbd397289c F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880 -F src/pragma.c d04725ac25387d9638919e197fb009f378e13af7bf899516979e54b3164e3602 +F src/pragma.c 49a04b2ec3199b7967c19b3239182d9c4e860726909ed80b7a3e21fe5bb9e6c4 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324 F src/prepare.c 9a141a1b02dca53beaa9771699d390aafcac01f5d1f1c0ae6e23ded8dcdb709a F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c @@ -526,7 +526,7 @@ F src/update.c 5404be9e840717323a69209190cdbc9d0d34adaedaaf1d1a1069babf2c4171c0 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 F src/util.c 5168013cfd937a695d23cce8c67cb07a3dda242d4cb812530ba1148b88e0f159 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739 -F src/vdbe.c 176c0897af0aedecd3abc9afaf7fa80eaa7cf5eaf62583de256a9961df474373 +F src/vdbe.c c3c3fe95d98df8757570fb265236cfdf31536d1025bf37c232237d1e422ae39b F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1 @@ -537,8 +537,8 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 839db09792fead5052bb35e533fa485e134913d547d05b5f42e537b73e63f07a -F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a +F src/wal.c 5ca528539a69edd6333dcab1d49e89d4f98efb6a23f0fda85bc52c4ec313db49 +F src/wal.h b6063e6be1b03389372f3f32240e99b8ab92c32cdd05aa0e31b30a21e4e41654 F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6 F src/where.c 049522adcf5426f1a8c3ed07be15e1ffa3266afd34e8e7bee64b63e2fbfad0b5 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971 @@ -1097,7 +1097,7 @@ F test/parser1.test 391b9bf9a229547a129c61ac345ed1a6f5eb1854 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test d911c9ba49088d22054a05dc73743f677872a92ac89288bcdeafa0ebf3f9c531 +F test/permutations.test 5fe80f417441ed6a69acd77f5190305dbf255cc2be214f8e36df4715c4f63f08 F test/pragma.test c31b5e98998c160a4c85b1e04f590655c67f2daa7f73854640cd120610e3ac15 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed @@ -1142,11 +1142,11 @@ F test/rowvalue9.test d8dd2c6ecac432dadaa79e41dc2434f007be1b6b F test/rowvaluefault.test 7cd9ccc6c2fbdd881672984087aad0491bb75504 F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 F test/run-wordcount.sh 891e89c4c2d16e629cd45951d4ed899ad12afc09 -F test/savepoint.test 1f8a6b1aea9a0d05837adc463d4bf47bd9d0f1c842f1c2a9caccd639baf34bf9 +F test/savepoint.test 69c56b891ce0ff28f1376b5516bf2b6a8b39d0430433216bfc496e72103baaa5 F test/savepoint2.test 9b8543940572a2f01a18298c3135ad0c9f4f67d7 F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd -F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 +F test/savepoint6.test 48a645a7bb3a59a6fcf06a7364cfe5b655c336760de39068f7c241b0fc80d963 F test/savepoint7.test cde525ea3075283eb950cdcdefe23ead4f700daa F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2 F test/scanstatus.test 5253c219e331318a437f436268e0e82345700285 @@ -1261,7 +1261,7 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptable2.test cd396beb41117a5302fff61767c35fa4270a0d5e F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc -F test/tester.tcl 9948bd856ce8a1c127f2f7900365387a42a917ce0dc87185bdd128fa5b11aff2 +F test/tester.tcl 71d30287dd22aae0eb9b07e62574336ae6a57e7a50e5dab320a604cfcca4b173 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1520,6 +1520,7 @@ F test/walro.test 4ab7ac01b77c2f894235c699d59e3e3c7f15a160 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e +F test/waltwo2.test 6e4d36500a20ff2d19761cf0e9a5d178e83d1798feda157ebc0681e01a35e56e F test/where.test f0c325563acde44f2c4ea6ba348e9e29f7121757 F test/where2.test 478d2170637b9211f593120648858593bf2445a1 F test/where3.test 54cdeb02157acc979de41530b804ae7b09552bf1 @@ -1655,7 +1656,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7c69f8f1089c3e3843fbf7ec37a897c849a3df822a4ce3b4fcde586adf991a3f -R 61b14913c42f4f709d3e3a3845316f97 -U drh -Z 37903880ad327c432f24949839963d82 +P 40964a4ef7565ea0ddf452f48cb22373d068528e07d40eefc008f2231c969422 +R 07b0a272ae21af8e4cb674b6ac44fa89 +T *branch * wal2 +T *sym-wal2 * +T +closed f04ded1d9b40d54463162264e37e6d92411d09427eea592ef05681035e2f2e64 +T -sym-trunk * +U dan +Z 07422baaaff21d5f15df6853d7e92b06 diff --git a/manifest.uuid b/manifest.uuid index c2c8cc6040..092af1e9bc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -40964a4ef7565ea0ddf452f48cb22373d068528e07d40eefc008f2231c969422 \ No newline at end of file +e2fc5c814cf6862d536aacb9eca66ecd31ba7e3e3033fa4c5564d533f4a18dfc \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index cadd711e45..b221b7cf9c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2993,10 +2993,10 @@ static int lockBtree(BtShared *pBt){ goto page1_init_failed; } #else - if( page1[18]>2 ){ + if( page1[18]>3 ){ pBt->btsFlags |= BTS_READ_ONLY; } - if( page1[19]>2 ){ + if( page1[19]>3 ){ goto page1_init_failed; } @@ -3008,9 +3008,9 @@ static int lockBtree(BtShared *pBt){ ** may not be the latest version - there may be a newer one in the log ** file. */ - if( page1[19]==2 && (pBt->btsFlags & BTS_NO_WAL)==0 ){ + if( page1[19]>=2 && (pBt->btsFlags & BTS_NO_WAL)==0 ){ int isOpen = 0; - rc = sqlite3PagerOpenWal(pBt->pPager, &isOpen); + rc = sqlite3PagerOpenWal(pBt->pPager, (page1[19]==3), &isOpen); if( rc!=SQLITE_OK ){ goto page1_init_failed; }else{ @@ -9866,7 +9866,7 @@ int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){ BtShared *pBt = pBtree->pBt; int rc; /* Return code */ - assert( iVersion==1 || iVersion==2 ); + assert( iVersion==1 || iVersion==2 || iVersion==3 ); /* If setting the version fields to 1, do not automatically open the ** WAL connection, even if the version fields are currently set to 2. diff --git a/src/os_unix.c b/src/os_unix.c index 4445104dd6..19ac47263f 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4163,7 +4163,7 @@ static int unixShmSystemLock( assert( sqlite3_mutex_held(pShmNode->mutex) || pShmNode->nRef==0 ); /* Shared locks never span more than one byte */ - assert( n==1 || lockType!=F_RDLCK ); + /* assert( n==1 || lockType!=F_RDLCK ); */ /* Locks are within range */ assert( n>=1 && n<=SQLITE_SHM_NLOCK ); @@ -4609,7 +4609,7 @@ static int unixShmLock( || flags==(SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE) || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED) || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE) ); - assert( n==1 || (flags & SQLITE_SHM_EXCLUSIVE)!=0 ); + /* assert( n==1 || (flags & SQLITE_SHM_EXCLUSIVE)!=0 ); */ assert( pShmNode->h>=0 || pDbFd->pInode->bProcessLock==1 ); assert( pShmNode->h<0 || pDbFd->pInode->bProcessLock==0 ); diff --git a/src/pager.c b/src/pager.c index a43614cdb4..5a2b01f77a 100644 --- a/src/pager.c +++ b/src/pager.c @@ -810,20 +810,6 @@ static const unsigned char aJournalMagic[] = { */ #define PAGER_MAX_PGNO 2147483647 -/* -** The argument to this macro is a file descriptor (type sqlite3_file*). -** Return 0 if it is not open, or non-zero (but not 1) if it is. -** -** This is so that expressions can be written as: -** -** if( isOpen(pPager->jfd) ){ ... -** -** instead of -** -** if( pPager->jfd->pMethods ){ ... -*/ -#define isOpen(pFd) ((pFd)->pMethods!=0) - /* ** Return true if this pager uses a write-ahead log to read page pgno. ** Return false if the pager reads pgno directly from the database. @@ -944,6 +930,7 @@ static int assert_pager_state(Pager *p){ assert( isOpen(p->jfd) || p->journalMode==PAGER_JOURNALMODE_OFF || p->journalMode==PAGER_JOURNALMODE_WAL + || p->journalMode==PAGER_JOURNALMODE_WAL2 ); } assert( pPager->dbOrigSize==pPager->dbFileSize ); @@ -958,6 +945,7 @@ static int assert_pager_state(Pager *p){ assert( isOpen(p->jfd) || p->journalMode==PAGER_JOURNALMODE_OFF || p->journalMode==PAGER_JOURNALMODE_WAL + || p->journalMode==PAGER_JOURNALMODE_WAL2 || (sqlite3OsDeviceCharacteristics(p->fd)&SQLITE_IOCAP_BATCH_ATOMIC) ); assert( pPager->dbOrigSize<=pPager->dbHintSize ); @@ -970,6 +958,7 @@ static int assert_pager_state(Pager *p){ assert( isOpen(p->jfd) || p->journalMode==PAGER_JOURNALMODE_OFF || p->journalMode==PAGER_JOURNALMODE_WAL + || p->journalMode==PAGER_JOURNALMODE_WAL2 || (sqlite3OsDeviceCharacteristics(p->fd)&SQLITE_IOCAP_BATCH_ATOMIC) ); break; @@ -2063,7 +2052,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ } pPager->journalOff = 0; }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST - || (pPager->exclusiveMode && pPager->journalMode!=PAGER_JOURNALMODE_WAL) + || (pPager->exclusiveMode && pPager->journalModetempFile); pPager->journalOff = 0; @@ -2077,7 +2066,8 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ assert( sqlite3JournalIsInMemory(pPager->jfd)==0 ); assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE || pPager->journalMode==PAGER_JOURNALMODE_MEMORY - || pPager->journalMode==PAGER_JOURNALMODE_WAL + || pPager->journalMode==PAGER_JOURNALMODE_WAL + || pPager->journalMode==PAGER_JOURNALMODE_WAL2 ); sqlite3OsClose(pPager->jfd); if( bDelete ){ @@ -3344,9 +3334,9 @@ static int pagerOpenWalIfPresent(Pager *pPager){ rc = sqlite3OsDelete(pPager->pVfs, pPager->zWal, 0); }else{ testcase( sqlite3PcachePagecount(pPager->pPCache)==0 ); - rc = sqlite3PagerOpenWal(pPager, 0); + rc = sqlite3PagerOpenWal(pPager, 0, 0); } - }else if( pPager->journalMode==PAGER_JOURNALMODE_WAL ){ + }else if( pPager->journalMode>=PAGER_JOURNALMODE_WAL ){ pPager->journalMode = PAGER_JOURNALMODE_DELETE; } } @@ -7245,6 +7235,7 @@ int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ || eMode==PAGER_JOURNALMODE_PERSIST || eMode==PAGER_JOURNALMODE_OFF || eMode==PAGER_JOURNALMODE_WAL + || eMode==PAGER_JOURNALMODE_WAL2 || eMode==PAGER_JOURNALMODE_MEMORY ); /* This routine is only called from the OP_JournalMode opcode, and @@ -7279,9 +7270,12 @@ int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ assert( (PAGER_JOURNALMODE_MEMORY & 5)==4 ); assert( (PAGER_JOURNALMODE_OFF & 5)==0 ); assert( (PAGER_JOURNALMODE_WAL & 5)==5 ); + assert( (PAGER_JOURNALMODE_WAL2 & 5)==4 ); assert( isOpen(pPager->fd) || pPager->exclusiveMode ); - if( !pPager->exclusiveMode && (eOld & 5)==1 && (eMode & 1)==0 ){ + if( !pPager->exclusiveMode && (eOld & 5)==1 && (eMode & 1)==0 + && eMode!=PAGER_JOURNALMODE_WAL2 /* TODO: fix this if possible */ + ){ /* In this case we would like to delete the journal file. If it is ** not possible, then that is not a problem. Deleting the journal file @@ -7443,7 +7437,7 @@ static int pagerExclusiveLock(Pager *pPager){ ** lock on the database file and use heap-memory to store the wal-index ** in. Otherwise, use the normal shared-memory. */ -static int pagerOpenWal(Pager *pPager){ +static int pagerOpenWal(Pager *pPager, int bWal2){ int rc = SQLITE_OK; assert( pPager->pWal==0 && pPager->tempFile==0 ); @@ -7464,7 +7458,7 @@ static int pagerOpenWal(Pager *pPager){ if( rc==SQLITE_OK ){ rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, pPager->zWal, pPager->exclusiveMode, - pPager->journalSizeLimit, &pPager->pWal + pPager->journalSizeLimit, bWal2, &pPager->pWal ); } pagerFixMaplimit(pPager); @@ -7490,6 +7484,7 @@ static int pagerOpenWal(Pager *pPager){ */ int sqlite3PagerOpenWal( Pager *pPager, /* Pager object */ + int bWal2, /* Open in wal2 mode if not already open */ int *pbOpen /* OUT: Set to true if call is a no-op */ ){ int rc = SQLITE_OK; /* Return code */ @@ -7506,9 +7501,9 @@ int sqlite3PagerOpenWal( /* Close any rollback journal previously open */ sqlite3OsClose(pPager->jfd); - rc = pagerOpenWal(pPager); + rc = pagerOpenWal(pPager, bWal2); if( rc==SQLITE_OK ){ - pPager->journalMode = PAGER_JOURNALMODE_WAL; + pPager->journalMode = bWal2?PAGER_JOURNALMODE_WAL2:PAGER_JOURNALMODE_WAL; pPager->eState = PAGER_OPEN; } }else{ @@ -7530,7 +7525,9 @@ int sqlite3PagerOpenWal( int sqlite3PagerCloseWal(Pager *pPager, sqlite3 *db){ int rc = SQLITE_OK; - assert( pPager->journalMode==PAGER_JOURNALMODE_WAL ); + assert( pPager->journalMode==PAGER_JOURNALMODE_WAL + || pPager->journalMode==PAGER_JOURNALMODE_WAL2 + ); /* If the log file is not already open, but does exist in the file-system, ** it may need to be checkpointed before the connection can switch to @@ -7545,7 +7542,7 @@ int sqlite3PagerCloseWal(Pager *pPager, sqlite3 *db){ ); } if( rc==SQLITE_OK && logexists ){ - rc = pagerOpenWal(pPager); + rc = pagerOpenWal(pPager, 0); } } diff --git a/src/pager.h b/src/pager.h index 126267bcc8..399fcd4de3 100644 --- a/src/pager.h +++ b/src/pager.h @@ -81,6 +81,23 @@ typedef struct PgHdr DbPage; #define PAGER_JOURNALMODE_TRUNCATE 3 /* Commit by truncating journal */ #define PAGER_JOURNALMODE_MEMORY 4 /* In-memory journal file */ #define PAGER_JOURNALMODE_WAL 5 /* Use write-ahead logging */ +#define PAGER_JOURNALMODE_WAL2 6 /* Use write-ahead logging mode 2 */ + +#define isWalMode(x) ((x)==PAGER_JOURNALMODE_WAL || (x)==PAGER_JOURNALMODE_WAL2) + +/* +** The argument to this macro is a file descriptor (type sqlite3_file*). +** Return 0 if it is not open, or non-zero (but not 1) if it is. +** +** This is so that expressions can be written as: +** +** if( isOpen(pPager->jfd) ){ ... +** +** instead of +** +** if( pPager->jfd->pMethods ){ ... +*/ +#define isOpen(pFd) ((pFd)->pMethods!=0) /* ** Flags that make up the mask passed to sqlite3PagerGet(). @@ -177,7 +194,7 @@ int sqlite3PagerSharedLock(Pager *pPager); int sqlite3PagerCheckpoint(Pager *pPager, sqlite3*, int, int*, int*); int sqlite3PagerWalSupported(Pager *pPager); int sqlite3PagerWalCallback(Pager *pPager); - int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); + int sqlite3PagerOpenWal(Pager *pPager, int, int *pisOpen); int sqlite3PagerCloseWal(Pager *pPager, sqlite3*); # ifdef SQLITE_DIRECT_OVERFLOW_READ int sqlite3PagerUseWal(Pager *pPager, Pgno); diff --git a/src/pragma.c b/src/pragma.c index 918b1d8131..b78940c971 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -260,7 +260,7 @@ const char *sqlite3JournalModename(int eMode){ static char * const azModeName[] = { "delete", "persist", "off", "truncate", "memory" #ifndef SQLITE_OMIT_WAL - , "wal" + , "wal", "wal2" #endif }; assert( PAGER_JOURNALMODE_DELETE==0 ); @@ -269,6 +269,7 @@ const char *sqlite3JournalModename(int eMode){ assert( PAGER_JOURNALMODE_TRUNCATE==3 ); assert( PAGER_JOURNALMODE_MEMORY==4 ); assert( PAGER_JOURNALMODE_WAL==5 ); + assert( PAGER_JOURNALMODE_WAL2==6 ); assert( eMode>=0 && eMode<=ArraySize(azModeName) ); if( eMode==ArraySize(azModeName) ) return 0; diff --git a/src/vdbe.c b/src/vdbe.c index 9687170bec..c7b5cf74c4 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -6325,6 +6325,7 @@ case OP_JournalMode: { /* out2 */ || eNew==PAGER_JOURNALMODE_OFF || eNew==PAGER_JOURNALMODE_MEMORY || eNew==PAGER_JOURNALMODE_WAL + || eNew==PAGER_JOURNALMODE_WAL2 || eNew==PAGER_JOURNALMODE_QUERY ); assert( pOp->p1>=0 && pOp->p1nDb ); @@ -6342,16 +6343,25 @@ case OP_JournalMode: { /* out2 */ /* Do not allow a transition to journal_mode=WAL for a database ** in temporary storage or if the VFS does not support shared memory */ - if( eNew==PAGER_JOURNALMODE_WAL + if( isWalMode(eNew) && (sqlite3Strlen30(zFilename)==0 /* Temp file */ || !sqlite3PagerWalSupported(pPager)) /* No shared-memory support */ ){ eNew = eOld; } - if( (eNew!=eOld) - && (eOld==PAGER_JOURNALMODE_WAL || eNew==PAGER_JOURNALMODE_WAL) - ){ + if( eNew!=eOld && (isWalMode(eNew) || isWalMode(eOld)) ){ + + /* Prevent changing directly to wal2 from wal mode. And vice versa. */ + if( isWalMode(eNew) && isWalMode(eOld) ){ + rc = SQLITE_ERROR; + sqlite3VdbeError(p, "cannot change from %s to %s mode", + sqlite3JournalModename(eOld), sqlite3JournalModename(eNew) + ); + goto abort_due_to_error; + } + + /* Prevent switching into or out of wal/wal2 mode mid-transaction */ if( !db->autoCommit || db->nVdbeRead>1 ){ rc = SQLITE_ERROR; sqlite3VdbeError(p, @@ -6359,31 +6369,33 @@ case OP_JournalMode: { /* out2 */ (eNew==PAGER_JOURNALMODE_WAL ? "into" : "out of") ); goto abort_due_to_error; - }else{ + } - if( eOld==PAGER_JOURNALMODE_WAL ){ - /* If leaving WAL mode, close the log file. If successful, the call - ** to PagerCloseWal() checkpoints and deletes the write-ahead-log - ** file. An EXCLUSIVE lock may still be held on the database file - ** after a successful return. - */ - rc = sqlite3PagerCloseWal(pPager, db); - if( rc==SQLITE_OK ){ - sqlite3PagerSetJournalMode(pPager, eNew); - } - }else if( eOld==PAGER_JOURNALMODE_MEMORY ){ - /* Cannot transition directly from MEMORY to WAL. Use mode OFF - ** as an intermediate */ - sqlite3PagerSetJournalMode(pPager, PAGER_JOURNALMODE_OFF); - } - - /* Open a transaction on the database file. Regardless of the journal - ** mode, this transaction always uses a rollback journal. + if( isWalMode(eOld) ){ + /* If leaving WAL mode, close the log file. If successful, the call + ** to PagerCloseWal() checkpoints and deletes the write-ahead-log + ** file. An EXCLUSIVE lock may still be held on the database file + ** after a successful return. */ - assert( sqlite3BtreeIsInTrans(pBt)==0 ); + rc = sqlite3PagerCloseWal(pPager, db); if( rc==SQLITE_OK ){ - rc = sqlite3BtreeSetVersion(pBt, (eNew==PAGER_JOURNALMODE_WAL ? 2 : 1)); + sqlite3PagerSetJournalMode(pPager, eNew); } + }else if( eOld==PAGER_JOURNALMODE_MEMORY ){ + /* Cannot transition directly from MEMORY to WAL. Use mode OFF + ** as an intermediate */ + sqlite3PagerSetJournalMode(pPager, PAGER_JOURNALMODE_OFF); + } + + /* Open a transaction on the database file. Regardless of the journal + ** mode, this transaction always uses a rollback journal. + */ + assert( sqlite3BtreeIsInTrans(pBt)==0 ); + if( rc==SQLITE_OK ){ + /* 1==rollback, 2==wal, 3==wal2 */ + rc = sqlite3BtreeSetVersion(pBt, + 1 + isWalMode(eNew) + (eNew==PAGER_JOURNALMODE_WAL2) + ); } } #endif /* ifndef SQLITE_OMIT_WAL */ diff --git a/src/wal.c b/src/wal.c index 9930b84421..36c04289a5 100644 --- a/src/wal.c +++ b/src/wal.c @@ -255,20 +255,19 @@ int sqlite3WalTrace = 0; #endif /* -** The maximum (and only) versions of the wal and wal-index formats -** that may be interpreted by this version of SQLite. +** Both the wal-file and the wal-index contain version fields +** indicating the current version of the system. If a client +** reads the header of a wal file (as part of recovery), or the +** wal-index (as part of opening a read transaction) and (a) the +** header checksum is correct but (b) the version field is not +** recognized, the operation fails with SQLITE_CANTOPEN. ** -** If a client begins recovering a WAL file and finds that (a) the checksum -** values in the wal-header are correct and (b) the version field is not -** WAL_MAX_VERSION, recovery fails and SQLite returns SQLITE_CANTOPEN. -** -** Similarly, if a client successfully reads a wal-index header (i.e. the -** checksum test is successful) and finds that the version field is not -** WALINDEX_MAX_VERSION, then no read-transaction is opened and SQLite -** returns SQLITE_CANTOPEN. +** Currently, clients support both version-1 ("journal_mode=wal") and +** version-2 ("journal_mode=wal2"). Legacy clients may support version-1 +** only. */ -#define WAL_MAX_VERSION 3007000 -#define WALINDEX_MAX_VERSION 3007000 +#define WAL_VERSION1 3007000 /* For "journal_mode=wal" */ +#define WAL_VERSION2 3021000 /* For "journal_mode=wal2" */ /* ** Indices of various locking bytes. WAL_NREADER is the number @@ -282,6 +281,35 @@ int sqlite3WalTrace = 0; #define WAL_READ_LOCK(I) (3+(I)) #define WAL_NREADER (SQLITE_SHM_NLOCK-3) +/* +** Values that may be stored in Wal.readLock in wal2 mode. +** +** In wal mode, the Wal.readLock member is set to -1 when no read-lock +** is held, or else is the index of the read-mark on which a lock is +** held. +** +** In wal2 mode, Wal.readLock must be set to one of the following values. +** A value of -1 still indicates that no read-lock is held, but the other +** values are symbolic. See the implementation of walLockReader() for +** details of how the symbols map to OS level locks. +*/ +#define WAL_LOCK_NONE -1 +#define WAL_LOCK_PART1 1 +#define WAL_LOCK_PART1_FULL2 2 +#define WAL_LOCK_PART2 3 +#define WAL_LOCK_PART2_FULL1 4 + +/* +** This constant is used in wal2 mode only. +** +** In wal2 mode, when committing a transaction, if the current wal file +** is sufficiently large and there are no conflicting locks held, the +** writer writes the new transaction into the start of the other wal +** file. Usually, "sufficiently large" is defined by the value configured +** using "PRAGMA journal_size_limit". However, if no such value has been +** configured, sufficiently large defaults to WAL_DEFAULT_WALSIZE frames. +*/ +#define WAL_DEFAULT_WALSIZE 1000 /* Object declarations */ typedef struct WalIndexHdr WalIndexHdr; @@ -301,21 +329,64 @@ typedef struct WalCkptInfo WalCkptInfo; ** The szPage value can be any power of 2 between 512 and 32768, inclusive. ** Or it can be 1 to represent a 65536-byte page. The latter case was ** added in 3.7.1 when support for 64K pages was added. +** +** WAL2 mode notes: Member variable mxFrame2 is only used in wal2 mode +** (when iVersion is set to WAL_VERSION2). The lower 31 bits store +** the maximum frame number in file *-wal2. The most significant bit +** is a flag - set if clients are currently appending to *-wal2, clear +** otherwise. */ struct WalIndexHdr { u32 iVersion; /* Wal-index version */ - u32 unused; /* Unused (padding) field */ + u32 mxFrame2; /* See "WAL2 mode notes" above */ u32 iChange; /* Counter incremented each transaction */ u8 isInit; /* 1 when initialized */ u8 bigEndCksum; /* True if checksums in WAL are big-endian */ u16 szPage; /* Database page size in bytes. 1==64K */ - u32 mxFrame; /* Index of last valid frame in the WAL */ + u32 mxFrame; /* Index of last valid frame in each WAL */ u32 nPage; /* Size of database in pages */ u32 aFrameCksum[2]; /* Checksum of last frame in log */ u32 aSalt[2]; /* Two salt values copied from WAL header */ u32 aCksum[2]; /* Checksum over all prior fields */ }; +/* +** The following macros and functions are get/set methods for the maximum +** frame numbers and current wal file values stored in the WalIndexHdr +** structure. These are helpful because of the unorthodox way in which +** the values are stored in wal2 mode (see above). They are equivalent +** to functions with the following signatures. +** +** u32 walidxGetMxFrame(WalIndexHdr*, int iWal); // get mxFrame +** void walidxSetMxFrame(WalIndexHdr*, int iWal, u32 val); // set mxFrame +** int walidxGetFile(WalIndexHdr*) // get file +** void walidxSetFile(WalIndexHdr*, int val); // set file +*/ +#define walidxGetMxFrame(pHdr, iWal) \ + ((iWal) ? ((pHdr)->mxFrame2 & 0x7FFFFFF) : (pHdr)->mxFrame) + +static void walidxSetMxFrame(WalIndexHdr *pHdr, int iWal, u32 mxFrame){ + if( iWal ){ + pHdr->mxFrame2 = (pHdr->mxFrame2 & 0x80000000) | mxFrame; + }else{ + pHdr->mxFrame = mxFrame; + } + assert( walidxGetMxFrame(pHdr, iWal)==mxFrame ); +} + +#define walidxGetFile(pHdr) ((pHdr)->mxFrame2 >> 31) + +#define walidxSetFile(pHdr, iWal) ( \ + (pHdr)->mxFrame2 = ((pHdr)->mxFrame2 & 0x7FFFFFFF) | ((iWal)<<31) \ +) + +/* +** Argument is a pointer to a Wal structure. Return true if the current +** cache of the wal-index header indicates "journal_mode=wal2" mode, or +** false otherwise. +*/ +#define isWalMode2(pWal) ((pWal)->hdr.iVersion==WAL_VERSION2) + /* ** A copy of the following object occurs in the wal-index immediately ** following the second copy of the WalIndexHdr. This object stores @@ -427,7 +498,7 @@ struct WalCkptInfo { struct Wal { sqlite3_vfs *pVfs; /* The VFS used to create pDbFd */ sqlite3_file *pDbFd; /* File handle for the database file */ - sqlite3_file *pWalFd; /* File handle for WAL file */ + sqlite3_file *apWalFd[2]; /* File handle for "*-wal" and "*-wal2" */ u32 iCallback; /* Value to pass to log callback (or 0) */ i64 mxWalSize; /* Truncate WAL to this size upon reset */ int nWiData; /* Size of array apWiData */ @@ -447,6 +518,7 @@ struct Wal { u32 minFrame; /* Ignore wal frames before this one */ u32 iReCksum; /* On commit, recalculate checksums from here */ const char *zWalName; /* Name of WAL file */ + char *zWalName2; /* Name of second WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ #ifdef SQLITE_DEBUG u8 lockError; /* True if a locking error has occurred */ @@ -454,6 +526,7 @@ struct Wal { #ifdef SQLITE_ENABLE_SNAPSHOT WalIndexHdr *pSnapshot; /* Start transaction here if not NULL */ #endif + int bWal2; /* bWal2 flag passed to WalOpen() */ }; /* @@ -667,7 +740,7 @@ static void walIndexWriteHdr(Wal *pWal){ assert( pWal->writeLock ); pWal->hdr.isInit = 1; - pWal->hdr.iVersion = WALINDEX_MAX_VERSION; + assert( pWal->hdr.iVersion==WAL_VERSION1||pWal->hdr.iVersion==WAL_VERSION2 ); walChecksumBytes(1, (u8*)&pWal->hdr, nCksum, 0, pWal->hdr.aCksum); memcpy((void*)&aHdr[1], (const void*)&pWal->hdr, sizeof(WalIndexHdr)); walShmBarrier(pWal); @@ -745,7 +818,7 @@ static int walDecodeFrame( } /* A frame is only valid if a checksum of the WAL header, - ** all prior frams, the first 16 bytes of this frame-header, + ** all prior frames, the first 16 bytes of this frame-header, ** and the frame-data matches the checksum in the last 8 ** bytes of this frame-header. */ @@ -831,6 +904,36 @@ static void walUnlockExclusive(Wal *pWal, int lockIdx, int n){ walLockName(lockIdx), n)); } +/* +** This function is used to take and release read-locks in wal2 mode. +** +** Use of WAL_READ_LOCK(x) slots for (1<=x<=4). +** +** 1) Partial read of *-wal-1 (blocks checkpointer from checkpointing) +** 2) Full read of *-wal-2 (blocks writer from writing) +** 3) Partial read of *-wal-2 (blocks checkpointer from checkpointing) +** 4) Full read of *-wal-1 (blocks writer from writing) +*/ +static int walLockReader(Wal *pWal, int eLock, int bLock){ + int i; /* Index of first readmark to lock */ + int n; /* Number of readmarks to lock */ + + assert( pWal->hdr.iVersion==WAL_VERSION2 ); + if( pWal->exclusiveMode ) return SQLITE_OK; + + switch( eLock ){ + case WAL_LOCK_PART1 : i = 1; n = 1; break; + case WAL_LOCK_PART1_FULL2: i = 1; n = 2; break; + case WAL_LOCK_PART2 : i = 3; n = 1; break; + case WAL_LOCK_PART2_FULL1: i = 3; n = 2; break; + default: assert( !"cannot happen" ); + } + + return sqlite3OsShmLock(pWal->pDbFd, WAL_READ_LOCK(i), n, + SQLITE_SHM_SHARED | (bLock ? SQLITE_SHM_LOCK : SQLITE_SHM_UNLOCK) + ); +} + /* ** Compute a hash on a page number. The resulting hash value must land ** between 0 and (HASHTABLE_NSLOT-1). The walHashNext() function advances @@ -891,6 +994,43 @@ static int walHashGet( return rc; } +static u32 walExternalEncode(int iWal, u32 iFrame){ + u32 iRet; + if( iWal ){ + iRet = HASHTABLE_NPAGE_ONE + iFrame; + iRet += ((iFrame-1) / HASHTABLE_NPAGE) * HASHTABLE_NPAGE; + }else{ + iRet = iFrame; + iFrame += HASHTABLE_NPAGE - HASHTABLE_NPAGE_ONE; + iRet += ((iFrame-1) / HASHTABLE_NPAGE) * HASHTABLE_NPAGE; + } + return iRet; +} + +/* +** Parameter iExternal is an external frame identifier. This function +** transforms it to a wal file number (0 or 1) and frame number within +** this wal file (reported via output parameter *piRead). +*/ +static int walExternalDecode(u32 iExternal, u32 *piRead){ + int iHash = (iExternal+HASHTABLE_NPAGE-HASHTABLE_NPAGE_ONE-1)/HASHTABLE_NPAGE; + + if( 0==(iHash & 0x01) ){ + /* A frame in wal file 0 */ + *piRead = (iExternal <= HASHTABLE_NPAGE_ONE) ? iExternal : + iExternal - (iHash/2) * HASHTABLE_NPAGE; + return 0; + } + if( iHash==0 ){ + *piRead = iExternal; + return 0; + }else{ + *piRead = iExternal - HASHTABLE_NPAGE_ONE - ((iHash-1)/2) * HASHTABLE_NPAGE; + } + + return (iHash % 2); +} + /* ** Return the number of the wal-index page that contains the hash-table ** and page-number array that contain entries corresponding to WAL frame @@ -908,6 +1048,22 @@ static int walFramePage(u32 iFrame){ return iHash; } +/* +** Return the index of the hash-table corresponding to frame iFrame of wal +** file iWal. +*/ +static int walFramePage2(int iWal, u32 iFrame){ + int iRet; + assert( iWal==0 || iWal==1 ); + assert( iFrame>0 ); + if( iWal==0 ){ + iRet = 2*((iFrame+HASHTABLE_NPAGE-HASHTABLE_NPAGE_ONE-1)/HASHTABLE_NPAGE); + }else{ + iRet = 1 + 2 * ((iFrame-1) / HASHTABLE_NPAGE); + } + return iRet; +} + /* ** Return the page number associated with frame iFrame in this WAL. */ @@ -919,6 +1075,10 @@ static u32 walFramePgno(Wal *pWal, u32 iFrame){ return pWal->apWiData[iHash][(iFrame-1-HASHTABLE_NPAGE_ONE)%HASHTABLE_NPAGE]; } +static u32 walFramePgno2(Wal *pWal, int iWal, u32 iFrame){ + return walFramePgno(pWal, walExternalEncode(iWal, iFrame)); +} + /* ** Remove entries from the hash table that point to WAL slots greater ** than pWal->hdr.mxFrame. @@ -938,26 +1098,36 @@ static void walCleanupHash(Wal *pWal){ int iLimit = 0; /* Zero values greater than this */ int nByte; /* Number of bytes to zero in aPgno[] */ int i; /* Used to iterate through aHash[] */ + int iWal = walidxGetFile(&pWal->hdr); + u32 mxFrame = walidxGetMxFrame(&pWal->hdr, iWal); + + u32 iExternal; + if( isWalMode2(pWal) ){ + iExternal = walExternalEncode(iWal, mxFrame); + }else{ + assert( iWal==0 ); + iExternal = mxFrame; + } assert( pWal->writeLock ); - testcase( pWal->hdr.mxFrame==HASHTABLE_NPAGE_ONE-1 ); - testcase( pWal->hdr.mxFrame==HASHTABLE_NPAGE_ONE ); - testcase( pWal->hdr.mxFrame==HASHTABLE_NPAGE_ONE+1 ); + testcase( mxFrame==HASHTABLE_NPAGE_ONE-1 ); + testcase( mxFrame==HASHTABLE_NPAGE_ONE ); + testcase( mxFrame==HASHTABLE_NPAGE_ONE+1 ); - if( pWal->hdr.mxFrame==0 ) return; + if( mxFrame==0 ) return; /* Obtain pointers to the hash-table and page-number array containing ** the entry that corresponds to frame pWal->hdr.mxFrame. It is guaranteed ** that the page said hash-table and array reside on is already mapped. */ - assert( pWal->nWiData>walFramePage(pWal->hdr.mxFrame) ); - assert( pWal->apWiData[walFramePage(pWal->hdr.mxFrame)] ); - walHashGet(pWal, walFramePage(pWal->hdr.mxFrame), &aHash, &aPgno, &iZero); + assert( pWal->nWiData>walFramePage(iExternal) ); + assert( pWal->apWiData[walFramePage(iExternal)] ); + walHashGet(pWal, walFramePage(iExternal), &aHash, &aPgno, &iZero); /* Zero all hash-table entries that correspond to frame numbers greater ** than pWal->hdr.mxFrame. */ - iLimit = pWal->hdr.mxFrame - iZero; + iLimit = iExternal - iZero; assert( iLimit>0 ); for(i=0; iiLimit ){ @@ -966,8 +1136,7 @@ static void walCleanupHash(Wal *pWal){ } /* Zero the entries in the aPgno array that correspond to frames with - ** frame numbers greater than pWal->hdr.mxFrame. - */ + ** frame numbers greater than pWal->hdr.mxFrame. */ nByte = (int)((char *)aHash - (char *)&aPgno[iLimit+1]); memset((void *)&aPgno[iLimit+1], 0, nByte); @@ -988,18 +1157,25 @@ static void walCleanupHash(Wal *pWal){ #endif /* SQLITE_ENABLE_EXPENSIVE_ASSERT */ } - /* ** Set an entry in the wal-index that will map database page number ** pPage into WAL frame iFrame. */ -static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ +static int walIndexAppend(Wal *pWal, int iWal, u32 iFrame, u32 iPage){ int rc; /* Return code */ u32 iZero = 0; /* One less than frame number of aPgno[1] */ volatile u32 *aPgno = 0; /* Page number array */ volatile ht_slot *aHash = 0; /* Hash table */ + u32 iExternal; + + if( isWalMode2(pWal) ){ + iExternal = walExternalEncode(iWal, iFrame); + }else{ + assert( iWal==0 ); + iExternal = iFrame; + } - rc = walHashGet(pWal, walFramePage(iFrame), &aHash, &aPgno, &iZero); + rc = walHashGet(pWal, walFramePage(iExternal), &aHash, &aPgno, &iZero); /* Assuming the wal-index file was successfully mapped, populate the ** page number array and hash table entry. @@ -1009,7 +1185,7 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ int idx; /* Value to write to hash-table slot */ int nCollide; /* Number of hash collisions */ - idx = iFrame - iZero; + idx = iExternal - iZero; assert( idx <= HASHTABLE_NSLOT/2 + 1 ); /* If this is the first entry to be added to this hash-table, zero the @@ -1071,6 +1247,133 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ return rc; } +/* +** Recover a single wal file - *-wal if iWal==0, or *-wal2 if iWal==1. +*/ +static int walIndexRecoverOne(Wal *pWal, int iWal, u32 *pnCkpt, int *pbZero){ + i64 nSize; /* Size of log file */ + u32 aFrameCksum[2] = {0, 0}; + int rc; + sqlite3_file *pWalFd = pWal->apWalFd[iWal]; + + assert( iWal==0 || iWal==1 ); + + memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); + sqlite3_randomness(8, pWal->hdr.aSalt); + + rc = sqlite3OsFileSize(pWalFd, &nSize); + if( rc==SQLITE_OK ){ + if( nSize>WAL_HDRSIZE ){ + u8 aBuf[WAL_HDRSIZE]; /* Buffer to load WAL header into */ + u8 *aFrame = 0; /* Malloc'd buffer to load entire frame */ + int szFrame; /* Number of bytes in buffer aFrame[] */ + u8 *aData; /* Pointer to data part of aFrame buffer */ + int iFrame; /* Index of last frame read */ + i64 iOffset; /* Next offset to read from log file */ + int szPage; /* Page size according to the log */ + u32 magic; /* Magic value read from WAL header */ + u32 version; /* Magic value read from WAL header */ + int isValid; /* True if this frame is valid */ + + /* Read in the WAL header. */ + rc = sqlite3OsRead(pWalFd, aBuf, WAL_HDRSIZE, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + /* If the database page size is not a power of two, or is greater than + ** SQLITE_MAX_PAGE_SIZE, conclude that the WAL file contains no valid + ** data. Similarly, if the 'magic' value is invalid, ignore the whole + ** WAL file. + */ + magic = sqlite3Get4byte(&aBuf[0]); + szPage = sqlite3Get4byte(&aBuf[8]); + if( (magic&0xFFFFFFFE)!=WAL_MAGIC + || szPage&(szPage-1) + || szPage>SQLITE_MAX_PAGE_SIZE + || szPage<512 + ){ + return SQLITE_OK; + } + pWal->hdr.bigEndCksum = (u8)(magic&0x00000001); + pWal->szPage = szPage; + + /* Verify that the WAL header checksum is correct */ + walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN, + aBuf, WAL_HDRSIZE-2*4, 0, pWal->hdr.aFrameCksum + ); + if( pWal->hdr.aFrameCksum[0]!=sqlite3Get4byte(&aBuf[24]) + || pWal->hdr.aFrameCksum[1]!=sqlite3Get4byte(&aBuf[28]) + ){ + return SQLITE_OK; + } + + memcpy(&pWal->hdr.aSalt, &aBuf[16], 8); + *pnCkpt = sqlite3Get4byte(&aBuf[12]); + + /* Verify that the version number on the WAL format is one that + ** are able to understand */ + version = sqlite3Get4byte(&aBuf[4]); + if( version!=WAL_VERSION1 && version!=WAL_VERSION2 ){ + return SQLITE_CANTOPEN_BKPT; + } + pWal->hdr.iVersion = version; + + /* Malloc a buffer to read frames into. */ + szFrame = szPage + WAL_FRAME_HDRSIZE; + aFrame = (u8 *)sqlite3_malloc64(szFrame); + if( !aFrame ){ + return SQLITE_NOMEM_BKPT; + } + aData = &aFrame[WAL_FRAME_HDRSIZE]; + + /* Read all frames from the log file. */ + iFrame = 0; + for(iOffset=WAL_HDRSIZE; (iOffset+szFrame)<=nSize; iOffset+=szFrame){ + u32 pgno; /* Database page number for frame */ + u32 nTruncate; /* dbsize field from frame header */ + + /* Read and decode the next log frame. */ + iFrame++; + rc = sqlite3OsRead(pWalFd, aFrame, szFrame, iOffset); + if( rc!=SQLITE_OK ) break; + isValid = walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame); + if( !isValid ) break; + rc = walIndexAppend(pWal, iWal, iFrame, pgno); + if( rc!=SQLITE_OK ) break; + + /* If nTruncate is non-zero, this is a commit record. */ + if( nTruncate ){ + pWal->hdr.mxFrame = iFrame; + pWal->hdr.nPage = nTruncate; + pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16)); + testcase( szPage<=32768 ); + testcase( szPage>=65536 ); + aFrameCksum[0] = pWal->hdr.aFrameCksum[0]; + aFrameCksum[1] = pWal->hdr.aFrameCksum[1]; + } + } + + sqlite3_free(aFrame); + }else if( pbZero && nSize==0 ){ + *pbZero = 1; + } + } + + pWal->hdr.aFrameCksum[0] = aFrameCksum[0]; + pWal->hdr.aFrameCksum[1] = aFrameCksum[1]; + + return rc; +} + +static int walOpenWal2(Wal *pWal){ + int rc = SQLITE_OK; + if( !isOpen(pWal->apWalFd[1]) ){ + int f = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); + rc = sqlite3OsOpen(pWal->pVfs, pWal->zWalName2, pWal->apWalFd[1], f, &f); + } + return rc; +} /* ** Recover the wal-index by reading the write-ahead log file. @@ -1084,10 +1387,12 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ */ static int walIndexRecover(Wal *pWal){ int rc; /* Return Code */ - i64 nSize; /* Size of log file */ - u32 aFrameCksum[2] = {0, 0}; int iLock; /* Lock offset to lock for checkpoint */ int nLock; /* Number of locks to hold */ + u32 nCkpt1 = 0xFFFFFFFF; + u32 nCkpt2 = 0xFFFFFFFF; + int bZero = 0; + WalIndexHdr hdr; /* Obtain an exclusive lock on all byte in the locking range not already ** locked by the caller. The caller is guaranteed to have locked the @@ -1107,147 +1412,116 @@ static int walIndexRecover(Wal *pWal){ } WALTRACE(("WAL%p: recovery begin...\n", pWal)); - memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); - - rc = sqlite3OsFileSize(pWal->pWalFd, &nSize); - if( rc!=SQLITE_OK ){ - goto recovery_error; - } - - if( nSize>WAL_HDRSIZE ){ - u8 aBuf[WAL_HDRSIZE]; /* Buffer to load WAL header into */ - u8 *aFrame = 0; /* Malloc'd buffer to load entire frame */ - int szFrame; /* Number of bytes in buffer aFrame[] */ - u8 *aData; /* Pointer to data part of aFrame buffer */ - int iFrame; /* Index of last frame read */ - i64 iOffset; /* Next offset to read from log file */ - int szPage; /* Page size according to the log */ - u32 magic; /* Magic value read from WAL header */ - u32 version; /* Magic value read from WAL header */ - int isValid; /* True if this frame is valid */ - - /* Read in the WAL header. */ - rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0); - if( rc!=SQLITE_OK ){ - goto recovery_error; - } - - /* If the database page size is not a power of two, or is greater than - ** SQLITE_MAX_PAGE_SIZE, conclude that the WAL file contains no valid - ** data. Similarly, if the 'magic' value is invalid, ignore the whole - ** WAL file. - */ - magic = sqlite3Get4byte(&aBuf[0]); - szPage = sqlite3Get4byte(&aBuf[8]); - if( (magic&0xFFFFFFFE)!=WAL_MAGIC - || szPage&(szPage-1) - || szPage>SQLITE_MAX_PAGE_SIZE - || szPage<512 - ){ - goto finished; - } - pWal->hdr.bigEndCksum = (u8)(magic&0x00000001); - pWal->szPage = szPage; - pWal->nCkpt = sqlite3Get4byte(&aBuf[12]); - memcpy(&pWal->hdr.aSalt, &aBuf[16], 8); - - /* Verify that the WAL header checksum is correct */ - walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN, - aBuf, WAL_HDRSIZE-2*4, 0, pWal->hdr.aFrameCksum - ); - if( pWal->hdr.aFrameCksum[0]!=sqlite3Get4byte(&aBuf[24]) - || pWal->hdr.aFrameCksum[1]!=sqlite3Get4byte(&aBuf[28]) - ){ - goto finished; + /* Recover the *-wal file. If a valid version-1 header is recovered + ** from it, do not open the *-wal2 file. Even if it exists. + ** + ** Otherwise, if the *-wal2 file exists or if the "wal2" flag was + ** specified when sqlite3WalOpen() was called, open and recover + ** the *-wal2 file. Except, if the *-wal file was zero bytes in size, + ** truncate the *-wal2 to zero bytes in size. + ** + ** After this block has run, if the *-wal2 file is open the system + ** starts up in VERSION2 mode. In this case pWal->hdr contains the + ** wal-index header considering only *-wal2. Stack variable hdr + ** contains the wal-index header considering only *-wal. The hash + ** tables are populated for both. + ** + ** Or, if the *-wal2 file is not open, start up in VERSION1 mode. + ** pWal->hdr is already populated. + */ + rc = walIndexRecoverOne(pWal, 0, &nCkpt1, &bZero); + assert( pWal->hdr.iVersion==0 + || pWal->hdr.iVersion==WAL_VERSION1 + || pWal->hdr.iVersion==WAL_VERSION2 + ); + if( rc==SQLITE_OK && pWal->hdr.iVersion!=WAL_VERSION1 ){ + int bOpen = 1; + sqlite3_vfs *pVfs = pWal->pVfs; + if( pWal->hdr.iVersion==0 && pWal->bWal2==0 ){ + rc = sqlite3OsAccess(pVfs, pWal->zWalName2, SQLITE_ACCESS_EXISTS, &bOpen); } - - /* Verify that the version number on the WAL format is one that - ** are able to understand */ - version = sqlite3Get4byte(&aBuf[4]); - if( version!=WAL_MAX_VERSION ){ - rc = SQLITE_CANTOPEN_BKPT; - goto finished; + if( rc==SQLITE_OK && bOpen ){ + rc = walOpenWal2(pWal); + if( rc==SQLITE_OK ){ + hdr = pWal->hdr; + rc = walIndexRecoverOne(pWal, 1, &nCkpt2, 0); + } } + } - /* Malloc a buffer to read frames into. */ - szFrame = szPage + WAL_FRAME_HDRSIZE; - aFrame = (u8 *)sqlite3_malloc64(szFrame); - if( !aFrame ){ - rc = SQLITE_NOMEM_BKPT; - goto recovery_error; - } - aData = &aFrame[WAL_FRAME_HDRSIZE]; + if( rc==SQLITE_OK ){ + volatile WalCkptInfo *pInfo; - /* Read all frames from the log file. */ - iFrame = 0; - for(iOffset=WAL_HDRSIZE; (iOffset+szFrame)<=nSize; iOffset+=szFrame){ - u32 pgno; /* Database page number for frame */ - u32 nTruncate; /* dbsize field from frame header */ + if( isOpen(pWal->apWalFd[1]) ){ + /* The case where *-wal2 may follow *-wal */ + if( nCkpt2<=0x0F && nCkpt2==nCkpt1+1 ){ + if( sqlite3Get4byte((u8*)(&pWal->hdr.aSalt[0]))==hdr.aFrameCksum[0] + && sqlite3Get4byte((u8*)(&pWal->hdr.aSalt[1]))==hdr.aFrameCksum[1] + ){ + walidxSetFile(&pWal->hdr, 1); + walidxSetMxFrame(&pWal->hdr, 1, pWal->hdr.mxFrame); + walidxSetMxFrame(&pWal->hdr, 0, hdr.mxFrame); + }else{ + pWal->hdr = hdr; + } + }else - /* Read and decode the next log frame. */ - iFrame++; - rc = sqlite3OsRead(pWal->pWalFd, aFrame, szFrame, iOffset); - if( rc!=SQLITE_OK ) break; - isValid = walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame); - if( !isValid ) break; - rc = walIndexAppend(pWal, iFrame, pgno); - if( rc!=SQLITE_OK ) break; + /* When *-wal may follow *-wal2 */ + if( (nCkpt2==0x0F && nCkpt1==0) || (nCkpt2<0x0F && nCkpt2==nCkpt1-1) ){ + if( sqlite3Get4byte((u8*)(&hdr.aSalt[0]))==pWal->hdr.aFrameCksum[0] + && sqlite3Get4byte((u8*)(&hdr.aSalt[1]))==pWal->hdr.aFrameCksum[1] + ){ + SWAP(WalIndexHdr, pWal->hdr, hdr); + walidxSetMxFrame(&pWal->hdr, 1, hdr.mxFrame); + } + }else - /* If nTruncate is non-zero, this is a commit record. */ - if( nTruncate ){ - pWal->hdr.mxFrame = iFrame; - pWal->hdr.nPage = nTruncate; - pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16)); - testcase( szPage<=32768 ); - testcase( szPage>=65536 ); - aFrameCksum[0] = pWal->hdr.aFrameCksum[0]; - aFrameCksum[1] = pWal->hdr.aFrameCksum[1]; + /* Fallback */ + if( nCkpt1<=nCkpt2 ){ + pWal->hdr = hdr; + }else{ + walidxSetFile(&pWal->hdr, 1); } + pWal->hdr.iVersion = WAL_VERSION2; + }else{ + pWal->hdr.iVersion = WAL_VERSION1; } - sqlite3_free(aFrame); - } - -finished: - if( rc==SQLITE_OK ){ - volatile WalCkptInfo *pInfo; - int i; - pWal->hdr.aFrameCksum[0] = aFrameCksum[0]; - pWal->hdr.aFrameCksum[1] = aFrameCksum[1]; walIndexWriteHdr(pWal); /* Reset the checkpoint-header. This is safe because this thread is ** currently holding locks that exclude all other readers, writers and - ** checkpointers. - */ + ** checkpointers. */ pInfo = walCkptInfo(pWal); - pInfo->nBackfill = 0; - pInfo->nBackfillAttempted = pWal->hdr.mxFrame; - pInfo->aReadMark[0] = 0; - for(i=1; iaReadMark[i] = READMARK_NOT_USED; - if( pWal->hdr.mxFrame ) pInfo->aReadMark[1] = pWal->hdr.mxFrame; + memset((void*)pInfo, 0, sizeof(WalCkptInfo)); + if( 0==isWalMode2(pWal) ){ + int i; + pInfo->nBackfillAttempted = pWal->hdr.mxFrame; + pInfo->aReadMark[0] = 0; + for(i=1; iaReadMark[i] = READMARK_NOT_USED; + if( pWal->hdr.mxFrame ) pInfo->aReadMark[1] = pWal->hdr.mxFrame; + } /* If more than one frame was recovered from the log file, report an ** event via sqlite3_log(). This is to help with identifying performance ** problems caused by applications routinely shutting down without - ** checkpointing the log file. - */ + ** checkpointing the log file. */ if( pWal->hdr.nPage ){ sqlite3_log(SQLITE_NOTICE_RECOVER_WAL, - "recovered %d frames from WAL file %s", - pWal->hdr.mxFrame, pWal->zWalName + "recovered (%d,%d) frames from WAL files %s[2] (%s mode)", + walidxGetMxFrame(&pWal->hdr, 0), walidxGetMxFrame(&pWal->hdr, 1), + pWal->zWalName, isWalMode2(pWal) ? "wal2" : "wal" ); } } -recovery_error: WALTRACE(("WAL%p: recovery %s\n", pWal, rc ? "failed" : "ok")); walUnlockExclusive(pWal, iLock, nLock); return rc; } /* -** Close an open wal-index. +** Close an open wal-index and wal files. */ static void walIndexClose(Wal *pWal, int isDelete){ if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){ @@ -1259,6 +1533,8 @@ static void walIndexClose(Wal *pWal, int isDelete){ }else{ sqlite3OsShmUnmap(pWal->pDbFd, isDelete); } + sqlite3OsClose(pWal->apWalFd[0]); + sqlite3OsClose(pWal->apWalFd[1]); } /* @@ -1282,11 +1558,14 @@ int sqlite3WalOpen( const char *zWalName, /* Name of the WAL file */ int bNoShm, /* True to run in heap-memory mode */ i64 mxWalSize, /* Truncate WAL to this size on reset */ + int bWal2, /* True to open in wal2 mode */ Wal **ppWal /* OUT: Allocated Wal handle */ ){ int rc; /* Return Code */ Wal *pRet; /* Object to allocate and return */ int flags; /* Flags passed to OsOpen() */ + int nWalName; /* Length of zWalName in bytes */ + int nByte; /* Bytes of space to allocate */ assert( zWalName && zWalName[0] ); assert( pDbFd ); @@ -1306,34 +1585,42 @@ int sqlite3WalOpen( assert( UNIX_SHM_BASE==WALINDEX_LOCK_OFFSET ); #endif + nWalName = sqlite3Strlen30(zWalName); + nByte = sizeof(Wal) + pVfs->szOsFile*2 + nWalName+2; /* Allocate an instance of struct Wal to return. */ *ppWal = 0; - pRet = (Wal*)sqlite3MallocZero(sizeof(Wal) + pVfs->szOsFile); + pRet = (Wal*)sqlite3MallocZero(nByte); if( !pRet ){ return SQLITE_NOMEM_BKPT; } pRet->pVfs = pVfs; - pRet->pWalFd = (sqlite3_file *)&pRet[1]; + pRet->apWalFd[0] = (sqlite3_file*)((char*)pRet+sizeof(Wal)); + pRet->apWalFd[1] = (sqlite3_file*)((char*)pRet+sizeof(Wal)+pVfs->szOsFile); pRet->pDbFd = pDbFd; - pRet->readLock = -1; + pRet->readLock = WAL_LOCK_NONE; pRet->mxWalSize = mxWalSize; pRet->zWalName = zWalName; pRet->syncHeader = 1; pRet->padToSectorBoundary = 1; pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); + pRet->bWal2 = bWal2; + + pRet->zWalName2 = (char*)pRet + sizeof(Wal) + 2*pVfs->szOsFile; + memcpy(pRet->zWalName2, zWalName, nWalName); + pRet->zWalName2[nWalName] = '2'; + pRet->zWalName2[nWalName+1] = '\0'; - /* Open file handle on the write-ahead log file. */ + /* Open a file handle on the first write-ahead log file. */ flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); - rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, flags, &flags); + rc = sqlite3OsOpen(pVfs, zWalName, pRet->apWalFd[0], flags, &flags); if( rc==SQLITE_OK && flags&SQLITE_OPEN_READONLY ){ pRet->readOnly = WAL_RDONLY; } if( rc!=SQLITE_OK ){ walIndexClose(pRet, 0); - sqlite3OsClose(pRet->pWalFd); sqlite3_free(pRet); }else{ int iDC = sqlite3OsDeviceCharacteristics(pDbFd); @@ -1546,30 +1833,40 @@ static void walIteratorFree(WalIterator *p){ ** pages in the WAL in ascending order. The caller must hold the checkpoint ** lock. ** -** On success, make *pp point to the newly allocated WalInterator object -** return SQLITE_OK. Otherwise, return an error code. If this routine -** returns an error, the value of *pp is undefined. +** On success, make *pp point to the newly allocated WalIterator object +** and return SQLITE_OK. Otherwise, return an error code. If this routine +** returns an error, the final value of *pp is undefined. ** ** The calling routine should invoke walIteratorFree() to destroy the ** WalIterator object when it has finished with it. */ -static int walIteratorInit(Wal *pWal, WalIterator **pp){ +static int walIteratorInit(Wal *pWal, int iWal, WalIterator **pp){ WalIterator *p; /* Return value */ int nSegment; /* Number of segments to merge */ u32 iLast; /* Last frame in log */ int nByte; /* Number of bytes to allocate */ int i; /* Iterator variable */ + int iLastSeg; /* Last hash table to iterate though */ ht_slot *aTmp; /* Temp space used by merge-sort */ int rc = SQLITE_OK; /* Return Code */ + int iMode = isWalMode2(pWal) ? 2 : 1; + + assert( isWalMode2(pWal) || iWal==0 ); /* This routine only runs while holding the checkpoint lock. And ** it only runs if there is actually content in the log (mxFrame>0). */ - assert( pWal->ckptLock && pWal->hdr.mxFrame>0 ); - iLast = pWal->hdr.mxFrame; + iLast = walidxGetMxFrame(&pWal->hdr, iWal); + assert( pWal->ckptLock && iLast>0 ); + + if( iMode==2 ){ + iLastSeg = walFramePage2(iWal, iLast); + }else{ + iLastSeg = walFramePage(iLast); + } + nSegment = 1 + (iLastSeg/iMode); /* Allocate space for the WalIterator object. */ - nSegment = walFramePage(iLast) + 1; nByte = sizeof(WalIterator) + (nSegment-1)*sizeof(struct WalSegment) + iLast*sizeof(ht_slot); @@ -1590,19 +1887,28 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ rc = SQLITE_NOMEM_BKPT; } - for(i=0; rc==SQLITE_OK && i=2 ); + }else{ + iZero = iExtZero; + } aPgno++; - if( (i+1)==nSegment ){ + if( i==iLastSeg ){ nEntry = (int)(iLast - iZero); }else{ nEntry = (int)((u32*)aHash - (u32*)aPgno); @@ -1614,10 +1920,10 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ aIndex[j] = (ht_slot)j; } walMergesort((u32 *)aPgno, aTmp, aIndex, &nEntry); - p->aSegment[i].iZero = iZero; - p->aSegment[i].nEntry = nEntry; - p->aSegment[i].aIndex = aIndex; - p->aSegment[i].aPgno = (u32 *)aPgno; + p->aSegment[i/iMode].iZero = iZero; + p->aSegment[i/iMode].nEntry = nEntry; + p->aSegment[i/iMode].aIndex = aIndex; + p->aSegment[i/iMode].aPgno = (u32 *)aPgno; } } sqlite3_free(aTmp); @@ -1678,6 +1984,7 @@ static void walRestartHdr(Wal *pWal, u32 salt1){ volatile WalCkptInfo *pInfo = walCkptInfo(pWal); int i; /* Loop counter */ u32 *aSalt = pWal->hdr.aSalt; /* Big-endian salt values */ + assert( isWalMode2(pWal)==0 ); pWal->nCkpt++; pWal->hdr.mxFrame = 0; sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); @@ -1739,15 +2046,30 @@ static int walCheckpoint( u32 mxPage; /* Max database page to write */ int i; /* Loop counter */ volatile WalCkptInfo *pInfo; /* The checkpoint status information */ + int bWal2 = isWalMode2(pWal); /* True for wal2 connections */ + int iCkpt = bWal2 ? !walidxGetFile(&pWal->hdr) : 0; + mxSafeFrame = walidxGetMxFrame(&pWal->hdr, iCkpt); szPage = walPagesize(pWal); testcase( szPage<=32768 ); testcase( szPage>=65536 ); pInfo = walCkptInfo(pWal); - if( pInfo->nBackfillhdr.mxFrame ){ + if( (bWal2==1 && pInfo->nBackfill==0 && mxSafeFrame) + || (bWal2==0 && pInfo->nBackfillapWalFd[iCkpt]; + mxPage = pWal->hdr.nPage; + + /* If this is a wal2 system, check for a reader holding a lock + ** preventing this checkpoint operation. If one is found, return + ** early. */ + if( bWal2 ){ + rc = walLockExclusive(pWal, WAL_READ_LOCK(1 + iCkpt*2), 1); + if( rc!=SQLITE_OK ) return rc; + } /* Allocate the iterator */ - rc = walIteratorInit(pWal, &pIter); + rc = walIteratorInit(pWal, iCkpt, &pIter); if( rc!=SQLITE_OK ){ return rc; } @@ -1757,52 +2079,53 @@ static int walCheckpoint( ** in the SQLITE_CHECKPOINT_PASSIVE mode. */ assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 ); - /* Compute in mxSafeFrame the index of the last frame of the WAL that is - ** safe to write into the database. Frames beyond mxSafeFrame might - ** overwrite database pages that are in use by active readers and thus - ** cannot be backfilled from the WAL. + + /* If this is a wal system (not wal2), compute in mxSafeFrame the index + ** of the last frame of the WAL that is safe to write into the database. + ** Frames beyond mxSafeFrame might overwrite database pages that are in + ** use by active readers and thus cannot be backfilled from the WAL. */ - mxSafeFrame = pWal->hdr.mxFrame; - mxPage = pWal->hdr.nPage; - for(i=1; iaReadMark[i]; - if( mxSafeFrame>y ){ - assert( y<=pWal->hdr.mxFrame ); - rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1); - if( rc==SQLITE_OK ){ - pInfo->aReadMark[i] = (i==1 ? mxSafeFrame : READMARK_NOT_USED); - walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); - }else if( rc==SQLITE_BUSY ){ - mxSafeFrame = y; - xBusy = 0; - }else{ - goto walcheckpoint_out; + if( bWal2==0 ){ + for(i=1; iaReadMark[i]; + if( mxSafeFrame>y ){ + assert( y<=pWal->hdr.mxFrame ); + rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1); + if( rc==SQLITE_OK ){ + pInfo->aReadMark[i] = (i==1 ? mxSafeFrame : READMARK_NOT_USED); + walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); + }else if( rc==SQLITE_BUSY ){ + mxSafeFrame = y; + xBusy = 0; + }else{ + goto walcheckpoint_out; + } } } } - if( pInfo->nBackfillnBackfillnBackfill; + assert( bWal2==0 || nBackfill==0 ); pInfo->nBackfillAttempted = mxSafeFrame; - /* Sync the WAL to disk */ - rc = sqlite3OsSync(pWal->pWalFd, CKPT_SYNC_FLAGS(sync_flags)); + /* Sync the wal file being checkpointed to disk */ + rc = sqlite3OsSync(pWalFd, CKPT_SYNC_FLAGS(sync_flags)); /* If the database may grow as a result of this checkpoint, hint - ** about the eventual size of the db file to the VFS layer. - */ + ** about the eventual size of the db file to the VFS layer. */ if( rc==SQLITE_OK ){ i64 nReq = ((i64)mxPage * szPage); rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); @@ -1811,21 +2134,27 @@ static int walCheckpoint( } } - /* Iterate through the contents of the WAL, copying data to the db file */ while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ i64 iOffset; - assert( walFramePgno(pWal, iFrame)==iDbpage ); + + assert( bWal2==1 || walFramePgno(pWal, iFrame)==iDbpage ); + assert( bWal2==0 || walFramePgno2(pWal, iCkpt, iFrame)==iDbpage ); + if( db->u1.isInterrupted ){ rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_INTERRUPT; break; } if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ){ + assert( bWal2==0 || iDbpage>mxPage ); continue; } iOffset = walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE; + WALTRACE(("WAL%p: checkpoint frame %d of wal %d to db page %d\n", + pWal, (int)iFrame, iCkpt, (int)iDbpage + )); /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */ - rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, iOffset); + rc = sqlite3OsRead(pWalFd, zBuf, szPage, iOffset); if( rc!=SQLITE_OK ) break; iOffset = (iDbpage-1)*(i64)szPage; testcase( IS_BIG_INT(iOffset) ); @@ -1833,23 +2162,22 @@ static int walCheckpoint( if( rc!=SQLITE_OK ) break; } - /* If work was actually accomplished... */ + /* Truncate the db file, sync the wal file and set the WalCkptInfo + ** flag to indicate that it has been checkpointed. */ + if( !bWal2 && rc==SQLITE_OK && mxSafeFrame==walIndexHdr(pWal)->mxFrame ){ + i64 szDb = pWal->hdr.nPage*(i64)szPage; + testcase( IS_BIG_INT(szDb) ); + rc = sqlite3OsTruncate(pWal->pDbFd, szDb); + } if( rc==SQLITE_OK ){ - if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){ - i64 szDb = pWal->hdr.nPage*(i64)szPage; - testcase( IS_BIG_INT(szDb) ); - rc = sqlite3OsTruncate(pWal->pDbFd, szDb); - if( rc==SQLITE_OK ){ - rc = sqlite3OsSync(pWal->pDbFd, CKPT_SYNC_FLAGS(sync_flags)); - } - } - if( rc==SQLITE_OK ){ - pInfo->nBackfill = mxSafeFrame; - } + rc = sqlite3OsSync(pWal->pDbFd, CKPT_SYNC_FLAGS(sync_flags)); + } + if( rc==SQLITE_OK ){ + pInfo->nBackfill = bWal2 ? 1 : mxSafeFrame; } /* Release the reader lock held while backfilling */ - walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1); + walUnlockExclusive(pWal, WAL_READ_LOCK(bWal2 ? 1 + iCkpt*2 : 0), 1); } if( rc==SQLITE_BUSY ){ @@ -1864,7 +2192,7 @@ static int walCheckpoint( ** until all readers have finished using the wal file. This ensures that ** the next process to write to the database restarts the wal file. */ - if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){ + if( bWal2==0 && rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){ assert( pWal->writeLock ); if( pInfo->nBackfillhdr.mxFrame ){ rc = SQLITE_BUSY; @@ -1889,7 +2217,7 @@ static int walCheckpoint( ** file-system. To avoid this, update the wal-index header to ** indicate that the log file contains zero valid frames. */ walRestartHdr(pWal, salt1); - rc = sqlite3OsTruncate(pWal->pWalFd, 0); + rc = sqlite3OsTruncate(pWal->apWalFd[0], 0); } walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); } @@ -1906,16 +2234,18 @@ static int walCheckpoint( ** it to exactly nMax bytes. If an error occurs while doing so, ignore it. */ static void walLimitSize(Wal *pWal, i64 nMax){ - i64 sz; - int rx; - sqlite3BeginBenignMalloc(); - rx = sqlite3OsFileSize(pWal->pWalFd, &sz); - if( rx==SQLITE_OK && (sz > nMax ) ){ - rx = sqlite3OsTruncate(pWal->pWalFd, nMax); - } - sqlite3EndBenignMalloc(); - if( rx ){ - sqlite3_log(rx, "cannot limit WAL size: %s", pWal->zWalName); + if( isWalMode2(pWal)==0 ){ + i64 sz; + int rx; + sqlite3BeginBenignMalloc(); + rx = sqlite3OsFileSize(pWal->apWalFd[0], &sz); + if( rx==SQLITE_OK && (sz > nMax ) ){ + rx = sqlite3OsTruncate(pWal->apWalFd[0], nMax); + } + sqlite3EndBenignMalloc(); + if( rx ){ + sqlite3_log(rx, "cannot limit WAL size: %s", pWal->zWalName); + } } } @@ -1944,39 +2274,50 @@ int sqlite3WalClose( if( zBuf!=0 && SQLITE_OK==(rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE)) ){ + int i; if( pWal->exclusiveMode==WAL_NORMAL_MODE ){ pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; } - rc = sqlite3WalCheckpoint(pWal, db, - SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0 - ); - if( rc==SQLITE_OK ){ - int bPersist = -1; - sqlite3OsFileControlHint( - pWal->pDbFd, SQLITE_FCNTL_PERSIST_WAL, &bPersist + for(i=0; rc==SQLITE_OK && i<2; i++){ + rc = sqlite3WalCheckpoint(pWal, db, + SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0 ); - if( bPersist!=1 ){ - /* Try to delete the WAL file if the checkpoint completed and - ** fsyned (rc==SQLITE_OK) and if we are not in persistent-wal - ** mode (!bPersist) */ - isDelete = 1; - }else if( pWal->mxWalSize>=0 ){ - /* Try to truncate the WAL file to zero bytes if the checkpoint - ** completed and fsynced (rc==SQLITE_OK) and we are in persistent - ** WAL mode (bPersist) and if the PRAGMA journal_size_limit is a - ** non-negative value (pWal->mxWalSize>=0). Note that we truncate - ** to zero bytes as truncating to the journal_size_limit might - ** leave a corrupt WAL file on disk. */ - walLimitSize(pWal, 0); + if( rc==SQLITE_OK ){ + int bPersist = -1; + sqlite3OsFileControlHint( + pWal->pDbFd, SQLITE_FCNTL_PERSIST_WAL, &bPersist + ); + if( bPersist!=1 ){ + /* Try to delete the WAL file if the checkpoint completed and + ** fsyned (rc==SQLITE_OK) and if we are not in persistent-wal + ** mode (!bPersist) */ + isDelete = 1; + }else if( pWal->mxWalSize>=0 ){ + /* Try to truncate the WAL file to zero bytes if the checkpoint + ** completed and fsynced (rc==SQLITE_OK) and we are in persistent + ** WAL mode (bPersist) and if the PRAGMA journal_size_limit is a + ** non-negative value (pWal->mxWalSize>=0). Note that we truncate + ** to zero bytes as truncating to the journal_size_limit might + ** leave a corrupt WAL file on disk. */ + walLimitSize(pWal, 0); + } } + + if( isWalMode2(pWal)==0 ) break; + + walCkptInfo(pWal)->nBackfill = 0; + walidxSetFile(&pWal->hdr, !walidxGetFile(&pWal->hdr)); + pWal->writeLock = 1; + walIndexWriteHdr(pWal); + pWal->writeLock = 0; } } walIndexClose(pWal, isDelete); - sqlite3OsClose(pWal->pWalFd); if( isDelete ){ sqlite3BeginBenignMalloc(); sqlite3OsDelete(pWal->pVfs, pWal->zWalName, 0); + sqlite3OsDelete(pWal->pVfs, pWal->zWalName2, 0); sqlite3EndBenignMalloc(); } WALTRACE(("WAL%p: closed\n", pWal)); @@ -2115,7 +2456,9 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ ** sure the wal-index was not constructed with some future format that ** this version of SQLite cannot understand. */ - if( badHdr==0 && pWal->hdr.iVersion!=WALINDEX_MAX_VERSION ){ + if( badHdr==0 + && pWal->hdr.iVersion!=WAL_VERSION1 && pWal->hdr.iVersion!=WAL_VERSION2 + ){ rc = SQLITE_CANTOPEN_BKPT; } @@ -2180,13 +2523,9 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ */ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ volatile WalCkptInfo *pInfo; /* Checkpoint information in wal-index */ - u32 mxReadMark; /* Largest aReadMark[] value */ - int mxI; /* Index of largest aReadMark[] value */ - int i; /* Loop counter */ int rc = SQLITE_OK; /* Return code */ - u32 mxFrame; /* Wal frame to lock to */ - assert( pWal->readLock<0 ); /* Not currently locked */ + assert( pWal->readLock==WAL_LOCK_NONE ); /* Not currently locked */ /* Take steps to avoid spinning forever if there is a protocol error. ** @@ -2248,131 +2587,156 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ } pInfo = walCkptInfo(pWal); - if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame + if( isWalMode2(pWal) ){ + int eLock = 1 + (walidxGetFile(&pWal->hdr)*2); + if( pInfo->nBackfill==0 ){ + eLock += walidxGetMxFrame(&pWal->hdr, !walidxGetFile(&pWal->hdr))>0; + } + rc = walLockReader(pWal, eLock, 1); + if( rc!=SQLITE_OK ){ + return rc; + } + + walShmBarrier(pWal); + if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){ + walLockReader(pWal, eLock, 0); + return WAL_RETRY; + }else{ + pWal->readLock = eLock; + } + assert( pWal->minFrame==0 && walFramePage(pWal->minFrame)==0 ); + }else{ + u32 mxReadMark; /* Largest aReadMark[] value */ + int mxI; /* Index of largest aReadMark[] value */ + int i; /* Loop counter */ + u32 mxFrame; /* Wal frame to lock to */ + + if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame #ifdef SQLITE_ENABLE_SNAPSHOT - && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0 - || 0==memcmp(&pWal->hdr, pWal->pSnapshot, sizeof(WalIndexHdr))) + && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0 + || 0==memcmp(&pWal->hdr, pWal->pSnapshot, sizeof(WalIndexHdr))) #endif - ){ - /* The WAL has been completely backfilled (or it is empty). - ** and can be safely ignored. - */ - rc = walLockShared(pWal, WAL_READ_LOCK(0)); - walShmBarrier(pWal); - if( rc==SQLITE_OK ){ - if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){ - /* It is not safe to allow the reader to continue here if frames - ** may have been appended to the log before READ_LOCK(0) was obtained. - ** When holding READ_LOCK(0), the reader ignores the entire log file, - ** which implies that the database file contains a trustworthy - ** snapshot. Since holding READ_LOCK(0) prevents a checkpoint from - ** happening, this is usually correct. - ** - ** However, if frames have been appended to the log (or if the log - ** is wrapped and written for that matter) before the READ_LOCK(0) - ** is obtained, that is not necessarily true. A checkpointer may - ** have started to backfill the appended frames but crashed before - ** it finished. Leaving a corrupt image in the database file. - */ - walUnlockShared(pWal, WAL_READ_LOCK(0)); - return WAL_RETRY; + ){ + /* The WAL has been completely backfilled (or it is empty). + ** and can be safely ignored. + */ + rc = walLockShared(pWal, WAL_READ_LOCK(0)); + walShmBarrier(pWal); + if( rc==SQLITE_OK ){ + if( memcmp((void*)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){ + /* It is not safe to allow the reader to continue here if frames + ** may have been appended to the log before READ_LOCK(0) was obtained. + ** When holding READ_LOCK(0), the reader ignores the entire log file, + ** which implies that the database file contains a trustworthy + ** snapshot. Since holding READ_LOCK(0) prevents a checkpoint from + ** happening, this is usually correct. + ** + ** However, if frames have been appended to the log (or if the log + ** is wrapped and written for that matter) before the READ_LOCK(0) + ** is obtained, that is not necessarily true. A checkpointer may + ** have started to backfill the appended frames but crashed before + ** it finished. Leaving a corrupt image in the database file. + */ + walUnlockShared(pWal, WAL_READ_LOCK(0)); + return WAL_RETRY; + } + pWal->readLock = 0; + return SQLITE_OK; + }else if( rc!=SQLITE_BUSY ){ + return rc; } - pWal->readLock = 0; - return SQLITE_OK; - }else if( rc!=SQLITE_BUSY ){ - return rc; } - } - /* If we get this far, it means that the reader will want to use - ** the WAL to get at content from recent commits. The job now is - ** to select one of the aReadMark[] entries that is closest to - ** but not exceeding pWal->hdr.mxFrame and lock that entry. - */ - mxReadMark = 0; - mxI = 0; - mxFrame = pWal->hdr.mxFrame; + /* If we get this far, it means that the reader will want to use + ** the WAL to get at content from recent commits. The job now is + ** to select one of the aReadMark[] entries that is closest to + ** but not exceeding pWal->hdr.mxFrame and lock that entry. + */ + mxReadMark = 0; + mxI = 0; + mxFrame = pWal->hdr.mxFrame; #ifdef SQLITE_ENABLE_SNAPSHOT - if( pWal->pSnapshot && pWal->pSnapshot->mxFramepSnapshot->mxFrame; - } -#endif - for(i=1; iaReadMark[i]; - if( mxReadMark<=thisMark && thisMark<=mxFrame ){ - assert( thisMark!=READMARK_NOT_USED ); - mxReadMark = thisMark; - mxI = i; + if( pWal->pSnapshot && pWal->pSnapshot->mxFramepSnapshot->mxFrame; } - } - if( (pWal->readOnly & WAL_SHM_RDONLY)==0 - && (mxReadMarkaReadMark[i] = mxFrame; + u32 thisMark = pInfo->aReadMark[i]; + if( mxReadMark<=thisMark && thisMark<=mxFrame ){ + assert( thisMark!=READMARK_NOT_USED ); + mxReadMark = thisMark; mxI = i; - walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); - break; - }else if( rc!=SQLITE_BUSY ){ - return rc; } } - } - if( mxI==0 ){ - assert( rc==SQLITE_BUSY || (pWal->readOnly & WAL_SHM_RDONLY)!=0 ); - return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTLOCK; - } + if( (pWal->readOnly & WAL_SHM_RDONLY)==0 + && (mxReadMarkaReadMark[i] = mxFrame; + mxI = i; + walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); + break; + }else if( rc!=SQLITE_BUSY ){ + return rc; + } + } + } + if( mxI==0 ){ + assert( rc==SQLITE_BUSY || (pWal->readOnly & WAL_SHM_RDONLY)!=0 ); + return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTLOCK; + } - rc = walLockShared(pWal, WAL_READ_LOCK(mxI)); - if( rc ){ - return rc==SQLITE_BUSY ? WAL_RETRY : rc; - } - /* Now that the read-lock has been obtained, check that neither the - ** value in the aReadMark[] array or the contents of the wal-index - ** header have changed. - ** - ** It is necessary to check that the wal-index header did not change - ** between the time it was read and when the shared-lock was obtained - ** on WAL_READ_LOCK(mxI) was obtained to account for the possibility - ** that the log file may have been wrapped by a writer, or that frames - ** that occur later in the log than pWal->hdr.mxFrame may have been - ** copied into the database by a checkpointer. If either of these things - ** happened, then reading the database with the current value of - ** pWal->hdr.mxFrame risks reading a corrupted snapshot. So, retry - ** instead. - ** - ** Before checking that the live wal-index header has not changed - ** since it was read, set Wal.minFrame to the first frame in the wal - ** file that has not yet been checkpointed. This client will not need - ** to read any frames earlier than minFrame from the wal file - they - ** can be safely read directly from the database file. - ** - ** Because a ShmBarrier() call is made between taking the copy of - ** nBackfill and checking that the wal-header in shared-memory still - ** matches the one cached in pWal->hdr, it is guaranteed that the - ** checkpointer that set nBackfill was not working with a wal-index - ** header newer than that cached in pWal->hdr. If it were, that could - ** cause a problem. The checkpointer could omit to checkpoint - ** a version of page X that lies before pWal->minFrame (call that version - ** A) on the basis that there is a newer version (version B) of the same - ** page later in the wal file. But if version B happens to like past - ** frame pWal->hdr.mxFrame - then the client would incorrectly assume - ** that it can read version A from the database file. However, since - ** we can guarantee that the checkpointer that set nBackfill could not - ** see any pages past pWal->hdr.mxFrame, this problem does not come up. - */ - pWal->minFrame = pInfo->nBackfill+1; - walShmBarrier(pWal); - if( pInfo->aReadMark[mxI]!=mxReadMark - || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) - ){ - walUnlockShared(pWal, WAL_READ_LOCK(mxI)); - return WAL_RETRY; - }else{ - assert( mxReadMark<=pWal->hdr.mxFrame ); - pWal->readLock = (i16)mxI; + rc = walLockShared(pWal, WAL_READ_LOCK(mxI)); + if( rc ){ + return rc==SQLITE_BUSY ? WAL_RETRY : rc; + } + /* Now that the read-lock has been obtained, check that neither the + ** value in the aReadMark[] array or the contents of the wal-index + ** header have changed. + ** + ** It is necessary to check that the wal-index header did not change + ** between the time it was read and when the shared-lock was obtained + ** on WAL_READ_LOCK(mxI) was obtained to account for the possibility + ** that the log file may have been wrapped by a writer, or that frames + ** that occur later in the log than pWal->hdr.mxFrame may have been + ** copied into the database by a checkpointer. If either of these things + ** happened, then reading the database with the current value of + ** pWal->hdr.mxFrame risks reading a corrupted snapshot. So, retry + ** instead. + ** + ** Before checking that the live wal-index header has not changed + ** since it was read, set Wal.minFrame to the first frame in the wal + ** file that has not yet been checkpointed. This client will not need + ** to read any frames earlier than minFrame from the wal file - they + ** can be safely read directly from the database file. + ** + ** Because a ShmBarrier() call is made between taking the copy of + ** nBackfill and checking that the wal-header in shared-memory still + ** matches the one cached in pWal->hdr, it is guaranteed that the + ** checkpointer that set nBackfill was not working with a wal-index + ** header newer than that cached in pWal->hdr. If it were, that could + ** cause a problem. The checkpointer could omit to checkpoint + ** a version of page X that lies before pWal->minFrame (call that version + ** A) on the basis that there is a newer version (version B) of the same + ** page later in the wal file. But if version B happens to like past + ** frame pWal->hdr.mxFrame - then the client would incorrectly assume + ** that it can read version A from the database file. However, since + ** we can guarantee that the checkpointer that set nBackfill could not + ** see any pages past pWal->hdr.mxFrame, this problem does not come up. + */ + pWal->minFrame = pInfo->nBackfill+1; + walShmBarrier(pWal); + if( pInfo->aReadMark[mxI]!=mxReadMark + || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) + ){ + walUnlockShared(pWal, WAL_READ_LOCK(mxI)); + return WAL_RETRY; + }else{ + assert( mxReadMark<=pWal->hdr.mxFrame ); + pWal->readLock = (i16)mxI; + } } return rc; } @@ -2488,6 +2852,10 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ testcase( (rc&0xff)==SQLITE_IOERR ); testcase( rc==SQLITE_PROTOCOL ); testcase( rc==SQLITE_OK ); + + if( rc==SQLITE_OK && pWal->hdr.iVersion==WAL_VERSION2 ){ + rc = walOpenWal2(pWal); + } #ifdef SQLITE_ENABLE_SNAPSHOT if( rc==SQLITE_OK ){ @@ -2561,12 +2929,75 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ */ void sqlite3WalEndReadTransaction(Wal *pWal){ sqlite3WalEndWriteTransaction(pWal); - if( pWal->readLock>=0 ){ - walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); - pWal->readLock = -1; + if( pWal->readLock!=WAL_LOCK_NONE ){ + if( isWalMode2(pWal) ){ + (void)walLockReader(pWal, pWal->readLock, 0); + }else{ + walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); + } + pWal->readLock = WAL_LOCK_NONE; } } +/* Search hash table iHash for an entry matching page number +** pgno. Each call to this function searches a single hash table +** (each hash table indexes up to HASHTABLE_NPAGE frames). +** +** This code might run concurrently to the code in walIndexAppend() +** that adds entries to the wal-index (and possibly to this hash +** table). This means the value just read from the hash +** slot (aHash[iKey]) may have been added before or after the +** current read transaction was opened. Values added after the +** read transaction was opened may have been written incorrectly - +** i.e. these slots may contain garbage data. However, we assume +** that any slots written before the current read transaction was +** opened remain unmodified. +** +** For the reasons above, the if(...) condition featured in the inner +** loop of the following block is more stringent that would be required +** if we had exclusive access to the hash-table: +** +** (aPgno[iFrame]==pgno): +** This condition filters out normal hash-table collisions. +** +** (iFrame<=iLast): +** This condition filters out entries that were added to the hash +** table after the current read-transaction had started. +*/ +static int walSearchHash( + Wal *pWal, + u32 iLast, + int iHash, + Pgno pgno, + u32 *piRead +){ + volatile ht_slot *aHash; /* Pointer to hash table */ + volatile u32 *aPgno; /* Pointer to array of page numbers */ + u32 iZero; /* Frame number corresponding to aPgno[0] */ + int iKey; /* Hash slot index */ + int nCollide; /* Number of hash collisions remaining */ + int rc; /* Error code */ + + rc = walHashGet(pWal, iHash, &aHash, &aPgno, &iZero); + if( rc!=SQLITE_OK ){ + return rc; + } + nCollide = HASHTABLE_NSLOT; + for(iKey=walHash(pgno); aHash[iKey]; iKey=walNextHash(iKey)){ + u32 iFrame = aHash[iKey] + iZero; + if( iFrame<=iLast && iFrame>=pWal->minFrame && aPgno[aHash[iKey]]==pgno ){ + assert( iFrame>*piRead || CORRUPT_DB ); + *piRead = iFrame; + } + if( (nCollide--)==0 ){ + return SQLITE_CORRUPT_BKPT; + } + } + + return SQLITE_OK; +} + + /* ** Search the wal file for page pgno. If found, set *piRead to the frame that ** contains the page. Otherwise, if pgno is not in the wal file, set *piRead @@ -2580,80 +3011,74 @@ int sqlite3WalFindFrame( Pgno pgno, /* Database page number to read data for */ u32 *piRead /* OUT: Frame number (or zero) */ ){ + int bWal2 = isWalMode2(pWal); + int iApp = walidxGetFile(&pWal->hdr); + int rc = SQLITE_OK; u32 iRead = 0; /* If !=0, WAL frame to return data from */ - u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ + u32 iLast; /* Last frame in wal file */ int iHash; /* Used to loop through N hash tables */ - int iMinHash; /* This routine is only be called from within a read transaction. */ - assert( pWal->readLock>=0 || pWal->lockError ); + assert( pWal->readLock!=WAL_LOCK_NONE ); - /* If the "last page" field of the wal-index header snapshot is 0, then - ** no data will be read from the wal under any circumstances. Return early - ** in this case as an optimization. Likewise, if pWal->readLock==0, - ** then the WAL is ignored by the reader so return early, as if the - ** WAL were empty. - */ - if( iLast==0 || pWal->readLock==0 ){ - *piRead = 0; - return SQLITE_OK; + /* If this is a wal2 system, the client must have a partial-wal lock + ** on wal file iApp. Or if it is a wal system, iApp==0 must be true. */ + assert( bWal2==0 || iApp==1 + || pWal->readLock==WAL_LOCK_PART1 || pWal->readLock==WAL_LOCK_PART1_FULL2 + ); + assert( bWal2==0 || iApp==0 + || pWal->readLock==WAL_LOCK_PART2 || pWal->readLock==WAL_LOCK_PART2_FULL1 + ); + assert( bWal2 || iApp==0 ); + + /* Search the wal file that the client holds a partial lock on first */ + iLast = walidxGetMxFrame(&pWal->hdr, iApp); + if( iLast ){ + u32 iExternal = bWal2 ? walExternalEncode(iApp, iLast) : iLast; + int iMinHash = walFramePage(pWal->minFrame); + for(iHash=walFramePage(iExternal); + iHash>=iMinHash && iRead==0; + iHash-=(1+bWal2) + ){ + rc = walSearchHash(pWal, iExternal, iHash, pgno, &iRead); + if( rc!=SQLITE_OK ) break; + } } - /* Search the hash table or tables for an entry matching page number - ** pgno. Each iteration of the following for() loop searches one - ** hash table (each hash table indexes up to HASHTABLE_NPAGE frames). - ** - ** This code might run concurrently to the code in walIndexAppend() - ** that adds entries to the wal-index (and possibly to this hash - ** table). This means the value just read from the hash - ** slot (aHash[iKey]) may have been added before or after the - ** current read transaction was opened. Values added after the - ** read transaction was opened may have been written incorrectly - - ** i.e. these slots may contain garbage data. However, we assume - ** that any slots written before the current read transaction was - ** opened remain unmodified. - ** - ** For the reasons above, the if(...) condition featured in the inner - ** loop of the following block is more stringent that would be required - ** if we had exclusive access to the hash-table: - ** - ** (aPgno[iFrame]==pgno): - ** This condition filters out normal hash-table collisions. - ** - ** (iFrame<=iLast): - ** This condition filters out entries that were added to the hash - ** table after the current read-transaction had started. - */ - iMinHash = walFramePage(pWal->minFrame); - for(iHash=walFramePage(iLast); iHash>=iMinHash && iRead==0; iHash--){ - volatile ht_slot *aHash; /* Pointer to hash table */ - volatile u32 *aPgno; /* Pointer to array of page numbers */ - u32 iZero; /* Frame number corresponding to aPgno[0] */ - int iKey; /* Hash slot index */ - int nCollide; /* Number of hash collisions remaining */ - int rc; /* Error code */ - - rc = walHashGet(pWal, iHash, &aHash, &aPgno, &iZero); - if( rc!=SQLITE_OK ){ - return rc; - } - nCollide = HASHTABLE_NSLOT; - for(iKey=walHash(pgno); aHash[iKey]; iKey=walNextHash(iKey)){ - u32 iFrame = aHash[iKey] + iZero; - if( iFrame<=iLast && iFrame>=pWal->minFrame && aPgno[aHash[iKey]]==pgno ){ - assert( iFrame>iRead || CORRUPT_DB ); - iRead = iFrame; - } - if( (nCollide--)==0 ){ - return SQLITE_CORRUPT_BKPT; + /* If the requested page was not found, no error has occured, and + ** the client holds a full-wal lock on the other wal file, search it + ** too. */ + if( rc==SQLITE_OK && bWal2 && iRead==0 && ( + pWal->readLock==WAL_LOCK_PART1_FULL2 + || pWal->readLock==WAL_LOCK_PART2_FULL1 + )){ + iLast = walidxGetMxFrame(&pWal->hdr, !iApp); + if( iLast ){ + u32 iExternal = walExternalEncode(!iApp, iLast); + for(iHash=walFramePage2(!iApp, iLast); iHash>=0 && iRead==0; iHash -= 2){ + rc = walSearchHash(pWal, iExternal, iHash, pgno, &iRead); + if( rc!=SQLITE_OK ) break; } } } +#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) + if( iRead ){ + u32 iFrame; + int iWal = walExternalDecode(iRead, &iFrame); + WALTRACE(("WAL%p: page %d @ frame %d wal %d\n",pWal,(int)pgno,iFrame,iWal)); + }else{ + WALTRACE(("WAL%p: page %d not found\n", pWal, (int)pgno)); + } +#endif + #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT /* If expensive assert() statements are available, do a linear search ** of the wal-index file content. Make sure the results agree with the - ** result obtained using the hash indexes above. */ + ** result obtained using the hash indexes above. + ** + ** TODO: This is broken for wal2. + */ { u32 iRead2 = 0; u32 iTest; @@ -2679,26 +3104,40 @@ int sqlite3WalFindFrame( */ int sqlite3WalReadFrame( Wal *pWal, /* WAL handle */ - u32 iRead, /* Frame to read */ + u32 iExternal, /* Frame to read */ int nOut, /* Size of buffer pOut in bytes */ u8 *pOut /* Buffer to write page data to */ ){ int sz; + int iWal = 0; + u32 iRead; i64 iOffset; + + /* Figure out the page size */ sz = pWal->hdr.szPage; sz = (sz&0xfe00) + ((sz&0x0001)<<16); testcase( sz<=32768 ); testcase( sz>=65536 ); + + if( isWalMode2(pWal) ){ + /* Figure out which of the two wal files, and the frame within, that + ** iExternal refers to. */ + iWal = walExternalDecode(iExternal, &iRead); + }else{ + iRead = iExternal; + } + + WALTRACE(("WAL%p: reading frame %d wal %d\n", pWal, iRead, iWal)); iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ - return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset); + return sqlite3OsRead(pWal->apWalFd[iWal], pOut, (nOut>sz?sz:nOut), iOffset); } /* ** Return the size of the database in pages (or zero, if unknown). */ Pgno sqlite3WalDbsize(Wal *pWal){ - if( pWal && ALWAYS(pWal->readLock>=0) ){ + if( pWal && ALWAYS(pWal->readLock!=WAL_LOCK_NONE) ){ return pWal->hdr.nPage; } return 0; @@ -2723,7 +3162,7 @@ int sqlite3WalBeginWriteTransaction(Wal *pWal){ /* Cannot start a write transaction without first holding a read ** transaction. */ - assert( pWal->readLock>=0 ); + assert( pWal->readLock!=WAL_LOCK_NONE ); assert( pWal->writeLock==0 && pWal->iReCksum==0 ); if( pWal->readOnly ){ @@ -2781,18 +3220,21 @@ int sqlite3WalEndWriteTransaction(Wal *pWal){ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ int rc = SQLITE_OK; if( ALWAYS(pWal->writeLock) ){ - Pgno iMax = pWal->hdr.mxFrame; + int iWal = walidxGetFile(&pWal->hdr); + Pgno iMax = walidxGetMxFrame(&pWal->hdr, iWal); + Pgno iNew; Pgno iFrame; - + + assert( isWalMode2(pWal) || iWal==0 ); + /* Restore the clients cache of the wal-index header to the state it ** was in before the client began writing to the database. */ memcpy(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr)); + assert( walidxGetFile(&pWal->hdr)==iWal ); + iNew = walidxGetMxFrame(&pWal->hdr, walidxGetFile(&pWal->hdr)); - for(iFrame=pWal->hdr.mxFrame+1; - ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; - iFrame++ - ){ + for(iFrame=iNew+1; ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; iFrame++){ /* This call cannot fail. Unless the page for which the page number ** is passed as the second argument is (a) in the cache and ** (b) has an outstanding reference, then xUndo is either a no-op @@ -2804,10 +3246,16 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ ** page 1 is never written to the log until the transaction is ** committed. As a result, the call to xUndo may not fail. */ - assert( walFramePgno(pWal, iFrame)!=1 ); - rc = xUndo(pUndoCtx, walFramePgno(pWal, iFrame)); + Pgno pgno; + if( isWalMode2(pWal) ){ + pgno = walFramePgno2(pWal, iWal, iFrame); + }else{ + pgno = walFramePgno(pWal, iFrame); + } + assert( pgno!=1 ); + rc = xUndo(pUndoCtx, pgno); } - if( iMax!=pWal->hdr.mxFrame ) walCleanupHash(pWal); + if( iMax!=iNew ) walCleanupHash(pWal); } return rc; } @@ -2819,11 +3267,13 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ ** point in the event of a savepoint rollback (via WalSavepointUndo()). */ void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){ + int iWal = walidxGetFile(&pWal->hdr); assert( pWal->writeLock ); - aWalData[0] = pWal->hdr.mxFrame; + assert( isWalMode2(pWal) || iWal==0 ); + aWalData[0] = walidxGetMxFrame(&pWal->hdr, iWal); aWalData[1] = pWal->hdr.aFrameCksum[0]; aWalData[2] = pWal->hdr.aFrameCksum[1]; - aWalData[3] = pWal->nCkpt; + aWalData[3] = isWalMode2(pWal) ? iWal : pWal->nCkpt; } /* @@ -2834,21 +3284,24 @@ void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){ */ int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ int rc = SQLITE_OK; + int iWal = walidxGetFile(&pWal->hdr); + int iCmp = isWalMode2(pWal) ? iWal : pWal->nCkpt; assert( pWal->writeLock ); - assert( aWalData[3]!=pWal->nCkpt || aWalData[0]<=pWal->hdr.mxFrame ); + assert( isWalMode2(pWal) || iWal==0 ); + assert( aWalData[3]!=iCmp || aWalData[0]<=walidxGetMxFrame(&pWal->hdr,iWal) ); - if( aWalData[3]!=pWal->nCkpt ){ + if( aWalData[3]!=iCmp ){ /* This savepoint was opened immediately after the write-transaction ** was started. Right after that, the writer decided to wrap around ** to the start of the log. Update the savepoint values to match. */ aWalData[0] = 0; - aWalData[3] = pWal->nCkpt; + aWalData[3] = iCmp; } - if( aWalData[0]hdr.mxFrame ){ - pWal->hdr.mxFrame = aWalData[0]; + if( aWalData[0]hdr, iWal) ){ + walidxSetMxFrame(&pWal->hdr, iWal, aWalData[0]); pWal->hdr.aFrameCksum[0] = aWalData[1]; pWal->hdr.aFrameCksum[1] = aWalData[2]; walCleanupHash(pWal); @@ -2857,23 +3310,80 @@ int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ return rc; } +/* +** This function is used in wal2 mode. +** +** This function is called when writer pWal is just about to start +** writing out frames. The "other" wal file (wal file !pWal->hdr.iAppend) +** has been fully checkpointed. This function returns SQLITE_OK if there +** are no readers preventing the writer from switching to the other wal +** file. Or SQLITE_BUSY if there are. +*/ +static int walRestartOk(Wal *pWal){ + int rc; /* Return code */ + int iApp = walidxGetFile(&pWal->hdr); /* Current WAL file */ + + /* No reader can be doing a "partial" read of wal file !iApp - in that + ** case it would not have been possible to checkpoint the file. So + ** it is only necessary to test for "full" readers. See the comment + ** above walLockReader() function for exactly what this means in terms + ** of locks. */ + int i = (iApp==0) ? 2 : 4; + + rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1); + if( rc==SQLITE_OK ){ + walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); + } + return rc; +} + /* ** This function is called just before writing a set of frames to the log ** file (see sqlite3WalFrames()). It checks to see if, instead of appending -** to the current log file, it is possible to overwrite the start of the -** existing log file with the new frames (i.e. "reset" the log). If so, -** it sets pWal->hdr.mxFrame to 0. Otherwise, pWal->hdr.mxFrame is left -** unchanged. +** to the current log file, it is possible and desirable to switch to the +** other log file and write the new transaction to the start of it. +** If so, the wal-index header is updated accordingly - both in heap memory +** and in the *-shm file. ** ** SQLITE_OK is returned if no error is encountered (regardless of whether -** or not pWal->hdr.mxFrame is modified). An SQLite error code is returned +** or not the wal-index header is modified). An SQLite error code is returned ** if an error occurs. */ static int walRestartLog(Wal *pWal){ int rc = SQLITE_OK; - int cnt; - if( pWal->readLock==0 ){ + if( isWalMode2(pWal) ){ + int iApp = walidxGetFile(&pWal->hdr); + int nWalSize = WAL_DEFAULT_WALSIZE; + if( pWal->mxWalSize>0 ){ + nWalSize = (pWal->mxWalSize-WAL_HDRSIZE+pWal->szPage+WAL_FRAME_HDRSIZE-1) + / (pWal->szPage+WAL_FRAME_HDRSIZE); + nWalSize = MAX(nWalSize, 1); + } + + if( walidxGetMxFrame(&pWal->hdr, iApp)>=nWalSize ){ + volatile WalCkptInfo *pInfo = walCkptInfo(pWal); + if( walidxGetMxFrame(&pWal->hdr, !iApp)==0 || pInfo->nBackfill ){ + rc = walRestartOk(pWal); + if( rc==SQLITE_OK ){ + iApp = !iApp; + pWal->nCkpt++; + walidxSetFile(&pWal->hdr, iApp); + walidxSetMxFrame(&pWal->hdr, iApp, 0); + sqlite3Put4byte((u8*)&pWal->hdr.aSalt[0], pWal->hdr.aFrameCksum[0]); + sqlite3Put4byte((u8*)&pWal->hdr.aSalt[1], pWal->hdr.aFrameCksum[1]); + walIndexWriteHdr(pWal); + pInfo->nBackfill = 0; + walLockReader(pWal, pWal->readLock, 0); + pWal->readLock = iApp ? WAL_LOCK_PART2_FULL1 : WAL_LOCK_PART1_FULL2; + rc = walLockReader(pWal, pWal->readLock, 1); + }else if( rc==SQLITE_BUSY ){ + rc = SQLITE_OK; + } + } + } + }else if( pWal->readLock==0 ){ + int cnt; volatile WalCkptInfo *pInfo = walCkptInfo(pWal); assert( pInfo->nBackfill==pWal->hdr.mxFrame ); if( pInfo->nBackfill>0 ){ @@ -2897,7 +3407,7 @@ static int walRestartLog(Wal *pWal){ } } walUnlockShared(pWal, WAL_READ_LOCK(0)); - pWal->readLock = -1; + pWal->readLock = WAL_LOCK_NONE; cnt = 0; do{ int notUsed; @@ -2908,6 +3418,7 @@ static int walRestartLog(Wal *pWal){ testcase( rc==SQLITE_PROTOCOL ); testcase( rc==SQLITE_OK ); } + return rc; } @@ -2966,6 +3477,18 @@ static int walWriteOneFrame( int rc; /* Result code from subfunctions */ void *pData; /* Data actually written */ u8 aFrame[WAL_FRAME_HDRSIZE]; /* Buffer to assemble frame-header in */ + +#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) + { + int iWal = walidxGetFile(&p->pWal->hdr); + int iFrame = 1 + (iOffset / (WAL_FRAME_HDRSIZE + p->pWal->szPage)); + assert( p->pWal->apWalFd[iWal]==p->pFd ); + WALTRACE(("WAL%p: page %d written to frame %d of wal %d\n", + p->pWal, (int)pPage->pgno, iFrame, iWal + )); + } +#endif + #if defined(SQLITE_HAS_CODEC) if( (pData = sqlite3PagerCodec(pPage))==0 ) return SQLITE_NOMEM_BKPT; #else @@ -2988,13 +3511,15 @@ static int walWriteOneFrame( ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ static int walRewriteChecksums(Wal *pWal, u32 iLast){ - const int szPage = pWal->szPage;/* Database page size */ int rc = SQLITE_OK; /* Return code */ + const int szPage = pWal->szPage;/* Database page size */ u8 *aBuf; /* Buffer to load data from wal file into */ u8 aFrame[WAL_FRAME_HDRSIZE]; /* Buffer to assemble frame-headers in */ u32 iRead; /* Next frame to read from wal file */ i64 iCksumOff; + assert( isWalMode2(pWal)==0 ); + aBuf = sqlite3_malloc(szPage + WAL_FRAME_HDRSIZE); if( aBuf==0 ) return SQLITE_NOMEM_BKPT; @@ -3009,7 +3534,7 @@ static int walRewriteChecksums(Wal *pWal, u32 iLast){ }else{ iCksumOff = walFrameOffset(pWal->iReCksum-1, szPage) + 16; } - rc = sqlite3OsRead(pWal->pWalFd, aBuf, sizeof(u32)*2, iCksumOff); + rc = sqlite3OsRead(pWal->apWalFd[0], aBuf, sizeof(u32)*2, iCksumOff); pWal->hdr.aFrameCksum[0] = sqlite3Get4byte(aBuf); pWal->hdr.aFrameCksum[1] = sqlite3Get4byte(&aBuf[sizeof(u32)]); @@ -3017,14 +3542,14 @@ static int walRewriteChecksums(Wal *pWal, u32 iLast){ pWal->iReCksum = 0; for(; rc==SQLITE_OK && iRead<=iLast; iRead++){ i64 iOff = walFrameOffset(iRead, szPage); - rc = sqlite3OsRead(pWal->pWalFd, aBuf, szPage+WAL_FRAME_HDRSIZE, iOff); + rc = sqlite3OsRead(pWal->apWalFd[0], aBuf, szPage+WAL_FRAME_HDRSIZE, iOff); if( rc==SQLITE_OK ){ u32 iPgno, nDbSize; iPgno = sqlite3Get4byte(aBuf); nDbSize = sqlite3Get4byte(&aBuf[4]); walEncodeFrame(pWal, iPgno, nDbSize, &aBuf[WAL_FRAME_HDRSIZE], aFrame); - rc = sqlite3OsWrite(pWal->pWalFd, aFrame, sizeof(aFrame), iOff); + rc = sqlite3OsWrite(pWal->apWalFd[0], aFrame, sizeof(aFrame), iOff); } } @@ -3054,6 +3579,7 @@ int sqlite3WalFrames( WalWriter w; /* The writer */ u32 iFirst = 0; /* First frame that may be overwritten */ WalIndexHdr *pLive; /* Pointer to shared header */ + int iApp; assert( pList ); assert( pWal->writeLock ); @@ -3062,22 +3588,17 @@ int sqlite3WalFrames( ** nTruncate==0 then this frame set does not complete the transaction. */ assert( (isCommit!=0)==(nTruncate!=0) ); -#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) - { int cnt; for(cnt=0, p=pList; p; p=p->pDirty, cnt++){} - WALTRACE(("WAL%p: frame write begin. %d frames. mxFrame=%d. %s\n", - pWal, cnt, pWal->hdr.mxFrame, isCommit ? "Commit" : "Spill")); - } -#endif - pLive = (WalIndexHdr*)walIndexHdr(pWal); if( memcmp(&pWal->hdr, (void *)pLive, sizeof(WalIndexHdr))!=0 ){ - iFirst = pLive->mxFrame+1; + if( isWalMode2(pWal)==0 ){ + iFirst = pLive->mxFrame+1; + } } /* See if it is possible to write these frames into the start of the ** log file, instead of appending to it at pWal->hdr.mxFrame. */ - if( SQLITE_OK!=(rc = walRestartLog(pWal)) ){ + else if( SQLITE_OK!=(rc = walRestartLog(pWal)) ){ return rc; } @@ -3085,28 +3606,50 @@ int sqlite3WalFrames( ** header to the start of the WAL file. See comments at the top of ** this source file for a description of the WAL header format. */ - iFrame = pWal->hdr.mxFrame; + iApp = walidxGetFile(&pWal->hdr); + iFrame = walidxGetMxFrame(&pWal->hdr, iApp); + assert( iApp==0 || isWalMode2(pWal) ); + +#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) + { int cnt; for(cnt=0, p=pList; p; p=p->pDirty, cnt++){} + WALTRACE(("WAL%p: frame write begin. %d frames. iWal=%d. mxFrame=%d. %s\n", + pWal, cnt, iApp, iFrame, isCommit ? "Commit" : "Spill")); + } +#endif + if( iFrame==0 ){ + u32 iCkpt = 0; u8 aWalHdr[WAL_HDRSIZE]; /* Buffer to assemble wal-header in */ u32 aCksum[2]; /* Checksum for wal-header */ sqlite3Put4byte(&aWalHdr[0], (WAL_MAGIC | SQLITE_BIGENDIAN)); - sqlite3Put4byte(&aWalHdr[4], WAL_MAX_VERSION); + sqlite3Put4byte(&aWalHdr[4], pWal->hdr.iVersion); sqlite3Put4byte(&aWalHdr[8], szPage); - sqlite3Put4byte(&aWalHdr[12], pWal->nCkpt); - if( pWal->nCkpt==0 ) sqlite3_randomness(8, pWal->hdr.aSalt); + if( isWalMode2(pWal) ){ + if( walidxGetMxFrame(&pWal->hdr, !iApp)>0 ){ + u8 aPrev[4]; + rc = sqlite3OsRead(pWal->apWalFd[!iApp], aPrev, 4, 12); + if( rc!=SQLITE_OK ){ + return rc; + } + iCkpt = (sqlite3Get4byte(aPrev) + 1) & 0x0F; + } + }else{ + iCkpt = pWal->nCkpt; + } + sqlite3Put4byte(&aWalHdr[12], iCkpt); memcpy(&aWalHdr[16], pWal->hdr.aSalt, 8); walChecksumBytes(1, aWalHdr, WAL_HDRSIZE-2*4, 0, aCksum); sqlite3Put4byte(&aWalHdr[24], aCksum[0]); sqlite3Put4byte(&aWalHdr[28], aCksum[1]); - + pWal->szPage = szPage; pWal->hdr.bigEndCksum = SQLITE_BIGENDIAN; pWal->hdr.aFrameCksum[0] = aCksum[0]; pWal->hdr.aFrameCksum[1] = aCksum[1]; pWal->truncateOnCommit = 1; - rc = sqlite3OsWrite(pWal->pWalFd, aWalHdr, sizeof(aWalHdr), 0); + rc = sqlite3OsWrite(pWal->apWalFd[iApp], aWalHdr, sizeof(aWalHdr), 0); WALTRACE(("WAL%p: wal-header write %s\n", pWal, rc ? "failed" : "ok")); if( rc!=SQLITE_OK ){ return rc; @@ -3120,7 +3663,7 @@ int sqlite3WalFrames( ** https://sqlite.org/src/info/ff5be73dee */ if( pWal->syncHeader ){ - rc = sqlite3OsSync(pWal->pWalFd, CKPT_SYNC_FLAGS(sync_flags)); + rc = sqlite3OsSync(pWal->apWalFd[iApp], CKPT_SYNC_FLAGS(sync_flags)); if( rc ) return rc; } } @@ -3128,7 +3671,7 @@ int sqlite3WalFrames( /* Setup information needed to write frames into the WAL */ w.pWal = pWal; - w.pFd = pWal->pWalFd; + w.pFd = pWal->apWalFd[iApp]; w.iSyncPoint = 0; w.syncFlags = sync_flags; w.szPage = szPage; @@ -3158,7 +3701,7 @@ int sqlite3WalFrames( #else pData = p->pData; #endif - rc = sqlite3OsWrite(pWal->pWalFd, pData, szPage, iOff); + rc = sqlite3OsWrite(pWal->apWalFd[iApp], pData, szPage, iOff); if( rc ) return rc; p->flags &= ~PGHDR_WAL_APPEND; continue; @@ -3198,7 +3741,7 @@ int sqlite3WalFrames( if( isCommit && WAL_SYNC_FLAGS(sync_flags)!=0 ){ int bSync = 1; if( pWal->padToSectorBoundary ){ - int sectorSize = sqlite3SectorSize(pWal->pWalFd); + int sectorSize = sqlite3SectorSize(w.pFd); w.iSyncPoint = ((iOffset+sectorSize-1)/sectorSize)*sectorSize; bSync = (w.iSyncPoint==iOffset); testcase( bSync ); @@ -3233,16 +3776,16 @@ int sqlite3WalFrames( ** guarantees that there are no other writers, and no data that may ** be in use by existing readers is being overwritten. */ - iFrame = pWal->hdr.mxFrame; + iFrame = walidxGetMxFrame(&pWal->hdr, iApp); for(p=pList; p && rc==SQLITE_OK; p=p->pDirty){ if( (p->flags & PGHDR_WAL_APPEND)==0 ) continue; iFrame++; - rc = walIndexAppend(pWal, iFrame, p->pgno); + rc = walIndexAppend(pWal, iApp, iFrame, p->pgno); } while( rc==SQLITE_OK && nExtra>0 ){ iFrame++; nExtra--; - rc = walIndexAppend(pWal, iFrame, pLast->pgno); + rc = walIndexAppend(pWal, iApp, iFrame, pLast->pgno); } if( rc==SQLITE_OK ){ @@ -3250,7 +3793,7 @@ int sqlite3WalFrames( pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16)); testcase( szPage<=32768 ); testcase( szPage>=65536 ); - pWal->hdr.mxFrame = iFrame; + walidxSetMxFrame(&pWal->hdr, iApp, iFrame); if( isCommit ){ pWal->hdr.iChange++; pWal->hdr.nPage = nTruncate; @@ -3258,7 +3801,17 @@ int sqlite3WalFrames( /* If this is a commit, update the wal-index header too. */ if( isCommit ){ walIndexWriteHdr(pWal); - pWal->iCallback = iFrame; + if( isWalMode2(pWal) ){ + int iOther = !walidxGetFile(&pWal->hdr); + if( walidxGetMxFrame(&pWal->hdr, iOther) + && !walCkptInfo(pWal)->nBackfill + ){ + pWal->iCallback = walidxGetMxFrame(&pWal->hdr, 0); + pWal->iCallback += walidxGetMxFrame(&pWal->hdr, 1); + } + }else{ + pWal->iCallback = iFrame; + } } } @@ -3350,7 +3903,9 @@ int sqlite3WalCheckpoint( /* Copy data from the log to the database file. */ if( rc==SQLITE_OK ){ - if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ + if( (walPagesize(pWal)!=nBuf) + && (walidxGetMxFrame(&pWal->hdr, 0) || walidxGetMxFrame(&pWal->hdr, 1)) + ){ rc = SQLITE_CORRUPT_BKPT; }else{ rc = walCheckpoint(pWal, db, eMode2, xBusy2, pBusyArg, sync_flags, zBuf); @@ -3358,8 +3913,20 @@ int sqlite3WalCheckpoint( /* If no error occurred, set the output variables. */ if( rc==SQLITE_OK || rc==SQLITE_BUSY ){ - if( pnLog ) *pnLog = (int)pWal->hdr.mxFrame; - if( pnCkpt ) *pnCkpt = (int)(walCkptInfo(pWal)->nBackfill); + if( pnLog ){ + *pnLog = walidxGetMxFrame(&pWal->hdr,0)+walidxGetMxFrame(&pWal->hdr,1); + } + if( pnCkpt ){ + if( isWalMode2(pWal) ){ + if( (int)(walCkptInfo(pWal)->nBackfill) ){ + *pnCkpt = walidxGetMxFrame(&pWal->hdr, !walidxGetFile(&pWal->hdr)); + }else{ + *pnCkpt = 0; + } + }else{ + *pnCkpt = walCkptInfo(pWal)->nBackfill; + } + } } } @@ -3421,6 +3988,7 @@ int sqlite3WalCallback(Wal *pWal){ */ int sqlite3WalExclusiveMode(Wal *pWal, int op){ int rc; + assert( pWal->writeLock==0 ); assert( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE || op==-1 ); @@ -3430,13 +3998,18 @@ int sqlite3WalExclusiveMode(Wal *pWal, int op){ ** locks are taken in this case). Nor should the pager attempt to ** upgrade to exclusive-mode following such an error. */ - assert( pWal->readLock>=0 || pWal->lockError ); - assert( pWal->readLock>=0 || (op<=0 && pWal->exclusiveMode==0) ); + assert( pWal->readLock!=WAL_LOCK_NONE || pWal->lockError ); + assert( pWal->readLock!=WAL_LOCK_NONE || (op<=0 && pWal->exclusiveMode==0) ); if( op==0 ){ if( pWal->exclusiveMode ){ pWal->exclusiveMode = 0; - if( walLockShared(pWal, WAL_READ_LOCK(pWal->readLock))!=SQLITE_OK ){ + if( isWalMode2(pWal) ){ + rc = walLockReader(pWal, pWal->readLock, 1); + }else{ + rc = walLockShared(pWal, WAL_READ_LOCK(pWal->readLock)); + } + if( rc==SQLITE_OK ){ pWal->exclusiveMode = 1; } rc = pWal->exclusiveMode==0; @@ -3447,7 +4020,11 @@ int sqlite3WalExclusiveMode(Wal *pWal, int op){ }else if( op>0 ){ assert( pWal->exclusiveMode==0 ); assert( pWal->readLock>=0 ); - walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); + if( isWalMode2(pWal) ){ + walLockReader(pWal, pWal->readLock, 0); + }else{ + walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); + } pWal->exclusiveMode = 1; rc = 1; }else{ @@ -3531,7 +4108,7 @@ int sqlite3WalFramesize(Wal *pWal){ /* Return the sqlite3_file object for the WAL file */ sqlite3_file *sqlite3WalFile(Wal *pWal){ - return pWal->pWalFd; + return pWal->apWalFd[0]; } #endif /* #ifndef SQLITE_OMIT_WAL */ diff --git a/src/wal.h b/src/wal.h index d97300a684..29ddd7eb0b 100644 --- a/src/wal.h +++ b/src/wal.h @@ -26,7 +26,7 @@ #define CKPT_SYNC_FLAGS(X) (((X)>>2)&0x03) #ifdef SQLITE_OMIT_WAL -# define sqlite3WalOpen(x,y,z) 0 +# define sqlite3WalOpen(w,x,y,z) 0 # define sqlite3WalLimit(x,y) # define sqlite3WalClose(v,w,x,y,z) 0 # define sqlite3WalBeginReadTransaction(y,z) 0 @@ -55,7 +55,7 @@ typedef struct Wal Wal; /* Open and close a connection to a write-ahead log. */ -int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *, int, i64, Wal**); +int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *,int,i64,int,Wal**); int sqlite3WalClose(Wal *pWal, sqlite3*, int sync_flags, int, u8 *); /* Set the limiting size of a WAL file. */ diff --git a/test/permutations.test b/test/permutations.test index 5afc51cb7d..7fbf98d412 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -1019,6 +1019,23 @@ test_suite "wal" -description { fts3c.test fts3d.test fts3e.test fts3query.test } +test_suite "wal2" -description { + Run tests with journal_mode=WAL2 +} -initialize { + set ::G(savepoint6_iterations) 100 +} -shutdown { + unset -nocomplain ::G(savepoint6_iterations) +} -files { + savepoint.test savepoint2.test savepoint6.test + trans.test avtrans.test + + fts3aa.test fts3ab.test fts3ac.test fts3ad.test + fts3ae.test fts3af.test fts3ag.test fts3ah.test + fts3ai.test fts3aj.test fts3ak.test fts3al.test + fts3am.test fts3an.test fts3ao.test fts3b.test + fts3c.test fts3d.test fts3e.test fts3query.test +} + test_suite "rtree" -description { All R-tree related tests. Provides coverage of source file rtree.c. } -files [glob -nocomplain $::testdir/../ext/rtree/*.test] diff --git a/test/savepoint.test b/test/savepoint.test index eed8a9e702..f196f8d2fc 100644 --- a/test/savepoint.test +++ b/test/savepoint.test @@ -28,6 +28,7 @@ do_test savepoint-1.1 { RELEASE sp1; } } {} +wal_check_journal_mode savepoint-1.1 do_test savepoint-1.2 { execsql { SAVEPOINT sp1; diff --git a/test/savepoint6.test b/test/savepoint6.test index b1d0d46f5c..6b41ef2da9 100644 --- a/test/savepoint6.test +++ b/test/savepoint6.test @@ -15,6 +15,10 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl proc sql {zSql} { + if {0 && $::debug_op} { + puts stderr "$zSql ;" + flush stderr + } uplevel db eval [list $zSql] #puts stderr "$zSql ;" } @@ -67,11 +71,13 @@ proc x_to_y {x} { # delete_rows XVALUES # proc savepoint {zName} { + if {$::debug_op} { puts stderr "savepoint $zName" ; flush stderr } catch { sql "SAVEPOINT $zName" } lappend ::lSavepoint [list $zName [array get ::aEntry]] } proc rollback {zName} { + if {$::debug_op} { puts stderr "rollback $zName" ; flush stderr } catch { sql "ROLLBACK TO $zName" } for {set i [expr {[llength $::lSavepoint]-1}]} {$i>=0} {incr i -1} { set zSavepoint [lindex $::lSavepoint $i 0] @@ -89,6 +95,7 @@ proc rollback {zName} { } proc release {zName} { + if {$::debug_op} { puts stderr "release $zName" ; flush stderr } catch { sql "RELEASE $zName" } for {set i [expr {[llength $::lSavepoint]-1}]} {$i>=0} {incr i -1} { set zSavepoint [lindex $::lSavepoint $i 0] @@ -104,6 +111,7 @@ proc release {zName} { } proc insert_rows {lX} { + if {$::debug_op} { puts stderr "insert_rows $lX" ; flush stderr } foreach x $lX { set y [x_to_y $x] @@ -116,6 +124,7 @@ proc insert_rows {lX} { } proc delete_rows {lX} { + if {$::debug_op} { puts stderr "delete_rows $lX" ; flush stderr } foreach x $lX { # Update database [db] sql "DELETE FROM t1 WHERE x = $x" @@ -164,6 +173,11 @@ proc random_integers {nRes nRange} { } #------------------------------------------------------------------------- +set ::debug_op 0 +proc debug_ops {} { + set ::debug_op 1 +} + proc database_op {} { set i [expr int(rand()*2)] if {$i==0} { @@ -185,9 +199,6 @@ proc savepoint_op {} { set C [lindex $cmds [expr int(rand()*6)]] set N [lindex $names [expr int(rand()*5)]] - #puts stderr " $C $N ; " - #flush stderr - $C $N return ok } diff --git a/test/tester.tcl b/test/tester.tcl index 10a20a47d6..4294b00786 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -586,6 +586,7 @@ proc reset_db {} { forcedelete test.db forcedelete test.db-journal forcedelete test.db-wal + forcedelete test.db-wal2 sqlite3 db ./test.db set ::DB [sqlite3_connection_pointer db] if {[info exists ::SETUP_SQL]} { @@ -2053,17 +2054,32 @@ proc drop_all_indexes {{db db}} { # Returns true if this test should be run in WAL mode. False otherwise. # proc wal_is_wal_mode {} { - expr {[permutation] eq "wal"} + if {[permutation] eq "wal"} { return 1 } + if {[permutation] eq "wal2"} { return 2 } + return 0 } proc wal_set_journal_mode {{db db}} { - if { [wal_is_wal_mode] } { - $db eval "PRAGMA journal_mode = WAL" + switch -- [wal_is_wal_mode] { + 0 { + } + + 1 { + $db eval "PRAGMA journal_mode = WAL" + } + + 2 { + $db eval "PRAGMA journal_mode = WAL2" + } } } proc wal_check_journal_mode {testname {db db}} { if { [wal_is_wal_mode] } { $db eval { SELECT * FROM sqlite_master } - do_test $testname [list $db eval "PRAGMA main.journal_mode"] {wal} + set expected "wal" + if {[wal_is_wal_mode]==2} { + set expected "wal2" + } + do_test $testname [list $db eval "PRAGMA main.journal_mode"] $expected } } diff --git a/test/waltwo2.test b/test/waltwo2.test new file mode 100644 index 0000000000..0ab5636a06 --- /dev/null +++ b/test/waltwo2.test @@ -0,0 +1,127 @@ +# 2017 September 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL2" mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/malloc_common.tcl +source $testdir/wal_common.tcl + +set testprefix walsimple +ifcapable !wal {finish_test ; return } + +db close +foreach f [glob -nocomplain test.db*] { forcedelete $f } +sqlite3 db test.db + +do_execsql_test 1.0 { + CREATE TABLE t1(x, y); + PRAGMA journal_mode = wal2; +} {wal2} + +do_execsql_test 1.1 { + SELECT * FROM t1; +} {} + +do_execsql_test 1.2 { + INSERT INTO t1 VALUES(1, 2); +} {} + +do_execsql_test 1.3 { + SELECT * FROM t1; +} {1 2} + +do_test 1.4 { + sqlite3 db2 test.db + execsql { SELECT * FROM t1 } db2 +} {1 2} + +do_test 1.5 { + lsort [glob test.db*] +} {test.db test.db-shm test.db-wal test.db-wal2} + +do_test 1.6 { + db close + db2 close + sqlite3 db test.db + execsql { SELECT * FROM t1 } +} {1 2} + +do_execsql_test 1.7 { + PRAGMA journal_size_limit = 4000; + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); + INSERT INTO t1 VALUES(7, 8); + INSERT INTO t1 VALUES(9, 10); + INSERT INTO t1 VALUES(11, 12); + INSERT INTO t1 VALUES(13, 14); + INSERT INTO t1 VALUES(15, 16); + INSERT INTO t1 VALUES(17, 18); + SELECT * FROM t1; +} {4000 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18} + +do_test 1.8 { + sqlite3 db2 test.db + execsql { SELECT * FROM t1 } db2 +} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18} + +do_test 1.9 { + db close + db2 close + lsort [glob test.db*] +} {test.db} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b, c); + PRAGMA journal_mode = wal2; + PRAGMA journal_size_limit = 4000; +} {wal2 4000} + +proc wal_hook {DB nm nFrame} { + $DB eval { PRAGMA wal_checkpoint } +} +db wal_hook [list wal_hook db] + + +foreach js {4000 8000 12000} { + foreach NROW [list 100 200 300 400 500 600 1000] { + do_test 2.$js.$NROW.1 { + db eval "DELETE FROM t1" + db eval "PRAGMA journal_size_limit = $js" + set nTotal 0 + for {set i 0} {$i < $NROW} {incr i} { + db eval { INSERT INTO t1 VALUES($i, $i, randomblob(abs(random()%50))) } + incr nTotal $i + } + set {} {} + } {} + + do_test 2.$js.$NROW.2 { + sqlite3 db2 test.db + db2 eval { + PRAGMA integrity_check; + SELECT count(*), sum(b) FROM t1; + } + } [list ok $NROW $nTotal] + + db2 close + } +} + +finish_test + From 3400e78f385c21d66ee3107bd78b856285a71b6e Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 5 Oct 2017 18:14:46 +0000 Subject: [PATCH 074/179] Fix test case failures on this branch. FossilOrigin-Name: 16decc13af908087fb8aa34eeccf43e8da1b8f2e4b808028986d1ef08134c72c --- manifest | 22 +++++++++------------- manifest.uuid | 2 +- src/wal.c | 41 +++++++++++++++++++++++++---------------- test/corruptA.test | 2 +- test/rdonly.test | 6 +++--- test/uri.test | 4 ++-- 6 files changed, 41 insertions(+), 36 deletions(-) diff --git a/manifest b/manifest index f6dbc867fd..e9519e9d3f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sexperimental\smode\sthat\suses\stwo\swal\sfiles.\sActivated\susing\s"PRAGMA\njournal_mode\s=\swal2". -D 2017-10-04T20:57:14.949 +C Fix\stest\scase\sfailures\son\sthis\sbranch. +D 2017-10-05T18:14:46.713 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f @@ -537,7 +537,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 5ca528539a69edd6333dcab1d49e89d4f98efb6a23f0fda85bc52c4ec313db49 +F src/wal.c 3455865cef56441a90c3688b65e091df3343e18c36bd4ad9da1154ecb22f2216 F src/wal.h b6063e6be1b03389372f3f32240e99b8ab92c32cdd05aa0e31b30a21e4e41654 F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6 F src/where.c 049522adcf5426f1a8c3ed07be15e1ffa3266afd34e8e7bee64b63e2fbfad0b5 @@ -675,7 +675,7 @@ F test/corrupt6.test fc6a891716139665dae0073b6945e3670bf92568 F test/corrupt7.test b036f94bda4b0b23a2919bf717046ce9ecca4543 F test/corrupt8.test 2399dfe40d2c0c63af86706e30f3e6302a8d0516 F test/corrupt9.test 730a3db08d4ab9aa43392ea30d9c2b4879cbff85 -F test/corruptA.test 112f4b2ae0b95ebf3ea63718642fb969a93acea557ace3a307234d19c245989b +F test/corruptA.test 56e8f321adaf3411960e9d2c7136669d8e1a91cbde6cf401ea84e6d6c7ccbe10 F test/corruptB.test 73a8d6c0b9833697ecf16b63e3c5c05c945b5dec F test/corruptC.test 138ecb02188ed1a719b533d4a139568204039f72f00e07a8d30d920bd83122db F test/corruptD.test b3c205fac7952b1de645ce44bb02335cd9e3e040 @@ -1118,7 +1118,7 @@ F test/quote.test 215897dbe8de1a6f701265836d6601cc6ed103e6 F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736 -F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8 +F test/rdonly.test 21e99ee237265d0cf95a0c84b50c784e834acaa4ef05d92a27b262626a656682 F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/regexp2.test 40e894223b3d6672655481493f1be12012f2b33c F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 @@ -1456,7 +1456,7 @@ F test/unixexcl.test d936ba2b06794018e136418addd59a2354eeae97 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 F test/update2.test fffc92e72ae568fe048588762e650cd8ccbd8c8b6e4fe9099231766bfe4b51de -F test/uri.test 3481026f00ade6dfe8adb7acb6e1e47b04369568 +F test/uri.test a2becabcb9fe25d08d1ae49c0788f4a75dda97bfe4c8641c2d04e224faf7a6e2 F test/uri2.test 9d3ba7a53ee167572d53a298ee4a5d38ec4a8fb7 F test/userauth01.test e740a2697a7b40d7c5003a7d7edaee16acd349a9 F test/utf16align.test 54cd35a27c005a9b6e7815d887718780b6a462ae @@ -1656,11 +1656,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 40964a4ef7565ea0ddf452f48cb22373d068528e07d40eefc008f2231c969422 -R 07b0a272ae21af8e4cb674b6ac44fa89 -T *branch * wal2 -T *sym-wal2 * -T +closed f04ded1d9b40d54463162264e37e6d92411d09427eea592ef05681035e2f2e64 -T -sym-trunk * +P e2fc5c814cf6862d536aacb9eca66ecd31ba7e3e3033fa4c5564d533f4a18dfc +R 69b71d3158015f2955b522d7c6fc4b0d U dan -Z 07422baaaff21d5f15df6853d7e92b06 +Z 23af8b038261b8f3163841b50db40cad diff --git a/manifest.uuid b/manifest.uuid index 092af1e9bc..2d0fb63ef8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e2fc5c814cf6862d536aacb9eca66ecd31ba7e3e3033fa4c5564d533f4a18dfc \ No newline at end of file +16decc13af908087fb8aa34eeccf43e8da1b8f2e4b808028986d1ef08134c72c \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 36c04289a5..db4fca8b94 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1507,11 +1507,18 @@ static int walIndexRecover(Wal *pWal){ ** problems caused by applications routinely shutting down without ** checkpointing the log file. */ if( pWal->hdr.nPage ){ - sqlite3_log(SQLITE_NOTICE_RECOVER_WAL, - "recovered (%d,%d) frames from WAL files %s[2] (%s mode)", - walidxGetMxFrame(&pWal->hdr, 0), walidxGetMxFrame(&pWal->hdr, 1), - pWal->zWalName, isWalMode2(pWal) ? "wal2" : "wal" - ); + if( isWalMode2(pWal) ){ + sqlite3_log(SQLITE_NOTICE_RECOVER_WAL, + "recovered (%d,%d) frames from WAL files %s[2] (%s mode)", + walidxGetMxFrame(&pWal->hdr, 0), walidxGetMxFrame(&pWal->hdr, 1), + pWal->zWalName, isWalMode2(pWal) ? "wal2" : "wal" + ); + }else{ + sqlite3_log(SQLITE_NOTICE_RECOVER_WAL, + "recovered %d frames from WAL file %s", + pWal->hdr.mxFrame, pWal->zWalName + ); + } } } @@ -2164,13 +2171,15 @@ static int walCheckpoint( /* Truncate the db file, sync the wal file and set the WalCkptInfo ** flag to indicate that it has been checkpointed. */ - if( !bWal2 && rc==SQLITE_OK && mxSafeFrame==walIndexHdr(pWal)->mxFrame ){ - i64 szDb = pWal->hdr.nPage*(i64)szPage; - testcase( IS_BIG_INT(szDb) ); - rc = sqlite3OsTruncate(pWal->pDbFd, szDb); - } - if( rc==SQLITE_OK ){ - rc = sqlite3OsSync(pWal->pDbFd, CKPT_SYNC_FLAGS(sync_flags)); + if( rc==SQLITE_OK && (bWal2 || mxSafeFrame==walIndexHdr(pWal)->mxFrame) ){ + if( !bWal2 ){ + i64 szDb = pWal->hdr.nPage*(i64)szPage; + testcase( IS_BIG_INT(szDb) ); + rc = sqlite3OsTruncate(pWal->pDbFd, szDb); + } + if( rc==SQLITE_OK ){ + rc = sqlite3OsSync(pWal->pDbFd, CKPT_SYNC_FLAGS(sync_flags)); + } } if( rc==SQLITE_OK ){ pInfo->nBackfill = bWal2 ? 1 : mxSafeFrame; @@ -4003,14 +4012,14 @@ int sqlite3WalExclusiveMode(Wal *pWal, int op){ if( op==0 ){ if( pWal->exclusiveMode ){ - pWal->exclusiveMode = 0; + pWal->exclusiveMode = WAL_NORMAL_MODE; if( isWalMode2(pWal) ){ rc = walLockReader(pWal, pWal->readLock, 1); }else{ rc = walLockShared(pWal, WAL_READ_LOCK(pWal->readLock)); } - if( rc==SQLITE_OK ){ - pWal->exclusiveMode = 1; + if( rc!=SQLITE_OK ){ + pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; } rc = pWal->exclusiveMode==0; }else{ @@ -4025,7 +4034,7 @@ int sqlite3WalExclusiveMode(Wal *pWal, int op){ }else{ walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); } - pWal->exclusiveMode = 1; + pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; rc = 1; }else{ rc = pWal->exclusiveMode==0; diff --git a/test/corruptA.test b/test/corruptA.test index 12d918615f..653b88b693 100644 --- a/test/corruptA.test +++ b/test/corruptA.test @@ -47,7 +47,7 @@ db close forcecopy test.db test.db-template set unreadable_version 02 -ifcapable wal { set unreadable_version 03 } +ifcapable wal { set unreadable_version 04 } do_test corruptA-2.1 { forcecopy test.db-template test.db hexio_write test.db 19 $unreadable_version ;# the read format number diff --git a/test/rdonly.test b/test/rdonly.test index 404c613b21..bdc70cdd70 100644 --- a/test/rdonly.test +++ b/test/rdonly.test @@ -41,7 +41,7 @@ do_test rdonly-1.1.1 { sqlite3_db_readonly db main } {0} -# Changes the write version from 1 to 3. Verify that the database +# Changes the write version from 1 to 4. Verify that the database # can be read but not written. # do_test rdonly-1.2 { @@ -49,7 +49,7 @@ do_test rdonly-1.2 { hexio_get_int [hexio_read test.db 18 1] } 1 do_test rdonly-1.3 { - hexio_write test.db 18 03 + hexio_write test.db 18 04 sqlite3 db test.db execsql { SELECT * FROM t1; @@ -83,7 +83,7 @@ do_test rdonly-1.5 { # the database is read-only until after it is locked. # set ro_version 02 -ifcapable wal { set ro_version 03 } +ifcapable wal { set ro_version 04 } do_test rdonly-1.6 { hexio_write test.db 18 $ro_version ; # write-version hexio_write test.db 24 11223344 ; # change-counter diff --git a/test/uri.test b/test/uri.test index 0931a25e51..b47f5fb05c 100644 --- a/test/uri.test +++ b/test/uri.test @@ -280,11 +280,11 @@ ifcapable wal { INSERT INTO t2 VALUES('x', 'y'); } lsort [array names ::T1] - } {test.db1 test.db1-journal test.db1-wal} + } {test.db1 test.db1-journal test.db1-wal test.db1-wal2} do_test 5.1.2 { lsort [array names ::T2] - } {test.db2 test.db2-journal test.db2-wal} + } {test.db2 test.db2-journal test.db2-wal test.db2-wal2} db close tvfs1 delete From 24f1b25c7b7b3b73033693c42bd9af956aaf3263 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 6 Oct 2017 13:43:42 +0000 Subject: [PATCH 075/179] Fix frame overwriting in wal2 mode. FossilOrigin-Name: a4b02bc9388226da21b3837a20c6c7eb0d13854dde62b7136e04f4978528dc71 --- manifest | 13 +++---- manifest.uuid | 2 +- src/wal.c | 67 ++++++++++++++++++++---------------- test/wal2rewrite.test | 80 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 126 insertions(+), 36 deletions(-) create mode 100644 test/wal2rewrite.test diff --git a/manifest b/manifest index e9519e9d3f..ebdae660fb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stest\scase\sfailures\son\sthis\sbranch. -D 2017-10-05T18:14:46.713 +C Fix\sframe\soverwriting\sin\swal2\smode. +D 2017-10-06T13:43:42.775 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f @@ -537,7 +537,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 3455865cef56441a90c3688b65e091df3343e18c36bd4ad9da1154ecb22f2216 +F src/wal.c 1fdb379a36877a21fa97220bbe075957c024a6d06716db692e1d35145738a5a1 F src/wal.h b6063e6be1b03389372f3f32240e99b8ab92c32cdd05aa0e31b30a21e4e41654 F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6 F src/where.c 049522adcf5426f1a8c3ed07be15e1ffa3266afd34e8e7bee64b63e2fbfad0b5 @@ -1492,6 +1492,7 @@ F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 F test/wal2.test 6ac39b94a284ebac6efb6be93b0cdfe73ee6083f129555e3144d8a615e9900ef +F test/wal2rewrite.test 467a66722b3e68b2c5c50f98fb62491c5f5e3f54b1d5728c7f8f1d5afe4976fe F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9 @@ -1656,7 +1657,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e2fc5c814cf6862d536aacb9eca66ecd31ba7e3e3033fa4c5564d533f4a18dfc -R 69b71d3158015f2955b522d7c6fc4b0d +P 16decc13af908087fb8aa34eeccf43e8da1b8f2e4b808028986d1ef08134c72c +R 756fa931da3c4e04a97cc0362e6290b7 U dan -Z 23af8b038261b8f3163841b50db40cad +Z 4ba52753cca3b375d5de2490acf53ff4 diff --git a/manifest.uuid b/manifest.uuid index 2d0fb63ef8..d9d56c6538 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -16decc13af908087fb8aa34eeccf43e8da1b8f2e4b808028986d1ef08134c72c \ No newline at end of file +a4b02bc9388226da21b3837a20c6c7eb0d13854dde62b7136e04f4978528dc71 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index db4fca8b94..54486f9969 100644 --- a/src/wal.c +++ b/src/wal.c @@ -3006,6 +3006,30 @@ static int walSearchHash( return SQLITE_OK; } +static int walSearchWal( + Wal *pWal, + int iWal, + Pgno pgno, + u32 *piRead +){ + int rc = SQLITE_OK; + int bWal2 = isWalMode2(pWal); + u32 iLast = walidxGetMxFrame(&pWal->hdr, iWal); + if( iLast ){ + int iHash; + int iMinHash = walFramePage(pWal->minFrame); + u32 iExternal = bWal2 ? walExternalEncode(iWal, iLast) : iLast; + assert( bWal2==0 || pWal->minFrame==0 ); + for(iHash=walFramePage(iExternal); + iHash>=iMinHash && *piRead==0; + iHash-=(1+bWal2) + ){ + rc = walSearchHash(pWal, iExternal, iHash, pgno, piRead); + if( rc!=SQLITE_OK ) break; + } + } + return rc; +} /* ** Search the wal file for page pgno. If found, set *piRead to the frame that @@ -3024,8 +3048,6 @@ int sqlite3WalFindFrame( int iApp = walidxGetFile(&pWal->hdr); int rc = SQLITE_OK; u32 iRead = 0; /* If !=0, WAL frame to return data from */ - u32 iLast; /* Last frame in wal file */ - int iHash; /* Used to loop through N hash tables */ /* This routine is only be called from within a read transaction. */ assert( pWal->readLock!=WAL_LOCK_NONE ); @@ -3041,18 +3063,8 @@ int sqlite3WalFindFrame( assert( bWal2 || iApp==0 ); /* Search the wal file that the client holds a partial lock on first */ - iLast = walidxGetMxFrame(&pWal->hdr, iApp); - if( iLast ){ - u32 iExternal = bWal2 ? walExternalEncode(iApp, iLast) : iLast; - int iMinHash = walFramePage(pWal->minFrame); - for(iHash=walFramePage(iExternal); - iHash>=iMinHash && iRead==0; - iHash-=(1+bWal2) - ){ - rc = walSearchHash(pWal, iExternal, iHash, pgno, &iRead); - if( rc!=SQLITE_OK ) break; - } - } + + rc = walSearchWal(pWal, iApp, pgno, &iRead); /* If the requested page was not found, no error has occured, and ** the client holds a full-wal lock on the other wal file, search it @@ -3061,14 +3073,7 @@ int sqlite3WalFindFrame( pWal->readLock==WAL_LOCK_PART1_FULL2 || pWal->readLock==WAL_LOCK_PART2_FULL1 )){ - iLast = walidxGetMxFrame(&pWal->hdr, !iApp); - if( iLast ){ - u32 iExternal = walExternalEncode(!iApp, iLast); - for(iHash=walFramePage2(!iApp, iLast); iHash>=0 && iRead==0; iHash -= 2){ - rc = walSearchHash(pWal, iExternal, iHash, pgno, &iRead); - if( rc!=SQLITE_OK ) break; - } - } + rc = walSearchWal(pWal, !iApp, pgno, &iRead); } #if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) @@ -3526,6 +3531,7 @@ static int walRewriteChecksums(Wal *pWal, u32 iLast){ u8 aFrame[WAL_FRAME_HDRSIZE]; /* Buffer to assemble frame-headers in */ u32 iRead; /* Next frame to read from wal file */ i64 iCksumOff; + sqlite3_file *pWalFd = pWal->apWalFd[walidxGetFile(&pWal->hdr)]; assert( isWalMode2(pWal)==0 ); @@ -3589,6 +3595,7 @@ int sqlite3WalFrames( u32 iFirst = 0; /* First frame that may be overwritten */ WalIndexHdr *pLive; /* Pointer to shared header */ int iApp; + int bWal2 = isWalMode2(pWal); assert( pList ); assert( pWal->writeLock ); @@ -3599,9 +3606,8 @@ int sqlite3WalFrames( pLive = (WalIndexHdr*)walIndexHdr(pWal); if( memcmp(&pWal->hdr, (void *)pLive, sizeof(WalIndexHdr))!=0 ){ - if( isWalMode2(pWal)==0 ){ - iFirst = pLive->mxFrame+1; - } + /* if( isWalMode2(pWal)==0 ) */ + iFirst = walidxGetMxFrame(pLive, walidxGetFile(pLive))+1; } /* See if it is possible to write these frames into the start of the @@ -3617,7 +3623,7 @@ int sqlite3WalFrames( */ iApp = walidxGetFile(&pWal->hdr); iFrame = walidxGetMxFrame(&pWal->hdr, iApp); - assert( iApp==0 || isWalMode2(pWal) ); + assert( iApp==0 || bWal2 ); #if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) { int cnt; for(cnt=0, p=pList; p; p=p->pDirty, cnt++){} @@ -3634,7 +3640,7 @@ int sqlite3WalFrames( sqlite3Put4byte(&aWalHdr[0], (WAL_MAGIC | SQLITE_BIGENDIAN)); sqlite3Put4byte(&aWalHdr[4], pWal->hdr.iVersion); sqlite3Put4byte(&aWalHdr[8], szPage); - if( isWalMode2(pWal) ){ + if( bWal2 ){ if( walidxGetMxFrame(&pWal->hdr, !iApp)>0 ){ u8 aPrev[4]; rc = sqlite3OsRead(pWal->apWalFd[!iApp], aPrev, 4, 12); @@ -3697,8 +3703,11 @@ int sqlite3WalFrames( ** checksums must be recomputed when the transaction is committed. */ if( iFirst && (p->pDirty || isCommit==0) ){ u32 iWrite = 0; - VVA_ONLY(rc =) sqlite3WalFindFrame(pWal, p->pgno, &iWrite); + VVA_ONLY(rc =) walSearchWal(pWal, iApp, p->pgno, &iWrite); assert( rc==SQLITE_OK || iWrite==0 ); + if( iWrite && bWal2 ){ + walExternalDecode(iWrite, &iWrite); + } if( iWrite>=iFirst ){ i64 iOff = walFrameOffset(iWrite, szPage) + WAL_FRAME_HDRSIZE; void *pData; @@ -3810,7 +3819,7 @@ int sqlite3WalFrames( /* If this is a commit, update the wal-index header too. */ if( isCommit ){ walIndexWriteHdr(pWal); - if( isWalMode2(pWal) ){ + if( bWal2 ){ int iOther = !walidxGetFile(&pWal->hdr); if( walidxGetMxFrame(&pWal->hdr, iOther) && !walCkptInfo(pWal)->nBackfill diff --git a/test/wal2rewrite.test b/test/wal2rewrite.test new file mode 100644 index 0000000000..dea4f8c4f5 --- /dev/null +++ b/test/wal2rewrite.test @@ -0,0 +1,80 @@ +# 2017 September 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL2" mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/malloc_common.tcl +source $testdir/wal_common.tcl + +set testprefix wal2rewrite +ifcapable !wal {finish_test ; return } + +proc filesize {filename} { + if {[file exists $filename]} { + return [file size $filename] + } + return 0 +} + +foreach {tn jrnlmode} { + 1 wal + 2 wal2 +} { + reset_db + execsql "PRAGMA journal_mode = $jrnlmode" + do_execsql_test $tn.1 { + PRAGMA journal_size_limit = 10000; + PRAGMA cache_size = 5; + PRAGMA wal_autocheckpoint = 10; + + CREATE TABLE t1(a INTEGER PRIMARY KEY, b INTEGER, c BLOB); + CREATE INDEX t1b ON t1(b); + CREATE INDEX t1c ON t1(c); + + WITH s(i) AS ( + SELECT 1 UNION SELECT i+1 FROM s WHERE i<10 + ) + INSERT INTO t1 SELECT i, i, randomblob(800) FROM s; + } {10000 10} + + for {set i 0} {$i < 4} {incr i} { + do_execsql_test $tn.$i.1 { + UPDATE t1 SET c=randomblob(800) WHERE (b%10)==5 AND ($i%2) + } + do_execsql_test $tn.$i.2 { + BEGIN; + UPDATE t1 SET b=b+10, c=randomblob(800); + UPDATE t1 SET b=b+10, c=randomblob(800); + UPDATE t1 SET b=b+10, c=randomblob(800); + UPDATE t1 SET b=b+10, c=randomblob(800); + UPDATE t1 SET b=b+10, c=randomblob(800); + UPDATE t1 SET b=b+10, c=randomblob(800); + UPDATE t1 SET b=b+10, c=randomblob(800); + UPDATE t1 SET b=b+10, c=randomblob(800); + UPDATE t1 SET b=b+10, c=randomblob(800); + UPDATE t1 SET b=b+10, c=randomblob(800); + } + execsql COMMIT + + do_test $tn.$i.3 { expr [filesize test.db-wal] < 100000 } 1 + do_test $tn.$i.4 { expr [filesize test.db-wal2] < 100000 } 1 + } + +} + + + +finish_test From bfcf8d6d675492c3bec7201781c7529284cebe3a Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 6 Oct 2017 14:08:39 +0000 Subject: [PATCH 076/179] Fix a bug in recovering wal2 mode databases introduced by the previous commit. FossilOrigin-Name: 9e1502e1b650217efc361732a3dfe686caa2e6352d040c73865f1faf09bf4591 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/wal.c | 6 +++--- test/wal2rewrite.test | 14 +++++++++++++- 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index ebdae660fb..2eb85d5859 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sframe\soverwriting\sin\swal2\smode. -D 2017-10-06T13:43:42.775 +C Fix\sa\sbug\sin\srecovering\swal2\smode\sdatabases\sintroduced\sby\sthe\sprevious\scommit. +D 2017-10-06T14:08:39.369 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f @@ -537,7 +537,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 1fdb379a36877a21fa97220bbe075957c024a6d06716db692e1d35145738a5a1 +F src/wal.c 287feccc5c6e886a5d747f5555382280e85a24904d0378940ef3869eb4273571 F src/wal.h b6063e6be1b03389372f3f32240e99b8ab92c32cdd05aa0e31b30a21e4e41654 F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6 F src/where.c 049522adcf5426f1a8c3ed07be15e1ffa3266afd34e8e7bee64b63e2fbfad0b5 @@ -1492,7 +1492,7 @@ F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 F test/wal2.test 6ac39b94a284ebac6efb6be93b0cdfe73ee6083f129555e3144d8a615e9900ef -F test/wal2rewrite.test 467a66722b3e68b2c5c50f98fb62491c5f5e3f54b1d5728c7f8f1d5afe4976fe +F test/wal2rewrite.test 6ca6f631ffcf871240beab5f02608913fd075c6d0d31310b026c8383c65c9f9c F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9 @@ -1657,7 +1657,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 16decc13af908087fb8aa34eeccf43e8da1b8f2e4b808028986d1ef08134c72c -R 756fa931da3c4e04a97cc0362e6290b7 +P a4b02bc9388226da21b3837a20c6c7eb0d13854dde62b7136e04f4978528dc71 +R 1b4fe3016ef19de5fbb230932d1cf085 U dan -Z 4ba52753cca3b375d5de2490acf53ff4 +Z bb0f71cc9a0d45c0ef66e315563912f3 diff --git a/manifest.uuid b/manifest.uuid index d9d56c6538..9ce9b5d7db 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a4b02bc9388226da21b3837a20c6c7eb0d13854dde62b7136e04f4978528dc71 \ No newline at end of file +9e1502e1b650217efc361732a3dfe686caa2e6352d040c73865f1faf09bf4591 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 54486f9969..2cc21039da 100644 --- a/src/wal.c +++ b/src/wal.c @@ -3549,7 +3549,7 @@ static int walRewriteChecksums(Wal *pWal, u32 iLast){ }else{ iCksumOff = walFrameOffset(pWal->iReCksum-1, szPage) + 16; } - rc = sqlite3OsRead(pWal->apWalFd[0], aBuf, sizeof(u32)*2, iCksumOff); + rc = sqlite3OsRead(pWalFd, aBuf, sizeof(u32)*2, iCksumOff); pWal->hdr.aFrameCksum[0] = sqlite3Get4byte(aBuf); pWal->hdr.aFrameCksum[1] = sqlite3Get4byte(&aBuf[sizeof(u32)]); @@ -3557,14 +3557,14 @@ static int walRewriteChecksums(Wal *pWal, u32 iLast){ pWal->iReCksum = 0; for(; rc==SQLITE_OK && iRead<=iLast; iRead++){ i64 iOff = walFrameOffset(iRead, szPage); - rc = sqlite3OsRead(pWal->apWalFd[0], aBuf, szPage+WAL_FRAME_HDRSIZE, iOff); + rc = sqlite3OsRead(pWalFd, aBuf, szPage+WAL_FRAME_HDRSIZE, iOff); if( rc==SQLITE_OK ){ u32 iPgno, nDbSize; iPgno = sqlite3Get4byte(aBuf); nDbSize = sqlite3Get4byte(&aBuf[4]); walEncodeFrame(pWal, iPgno, nDbSize, &aBuf[WAL_FRAME_HDRSIZE], aFrame); - rc = sqlite3OsWrite(pWal->apWalFd[0], aFrame, sizeof(aFrame), iOff); + rc = sqlite3OsWrite(pWalFd, aFrame, sizeof(aFrame), iOff); } } diff --git a/test/wal2rewrite.test b/test/wal2rewrite.test index dea4f8c4f5..7e3b7b17b5 100644 --- a/test/wal2rewrite.test +++ b/test/wal2rewrite.test @@ -71,8 +71,20 @@ foreach {tn jrnlmode} { do_test $tn.$i.3 { expr [filesize test.db-wal] < 100000 } 1 do_test $tn.$i.4 { expr [filesize test.db-wal2] < 100000 } 1 - } + set sum [db eval {SELECT sum(b), md5sum(c) FROM t1}] + + do_test $tn.$i.5 { + foreach f [glob -nocomplain test.db2*] {forcedelete $f} + foreach f [glob -nocomplain test.db*] { + forcecopy $f [string map {test.db test.db2} $f] + } + + sqlite3 db2 test.db2 + db2 eval {SELECT sum(b), md5sum(c) FROM t1} + } $sum + db2 close + } } From 5d122d308fd7439a5af69fcb297a303e4c6d892d Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 7 Oct 2017 13:37:04 +0000 Subject: [PATCH 077/179] Check in test file wal2simple.test. FossilOrigin-Name: 8932b2f1d7e6a26221ea3dea01000832b2d1eb17ac0b70ef6028f9286ae450a3 --- manifest | 12 +-- manifest.uuid | 2 +- test/wal2simple.test | 233 +++++++++++++++++++++++++++++++++++++++++++ test/waltwo2.test | 127 ----------------------- 4 files changed, 240 insertions(+), 134 deletions(-) create mode 100644 test/wal2simple.test delete mode 100644 test/waltwo2.test diff --git a/manifest b/manifest index 71878d3e69..5356b92b47 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\sinto\sthis\sbranch. -D 2017-10-06T14:25:25.410 +C Check\sin\stest\sfile\swal2simple.test. +D 2017-10-07T13:37:04.814 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f @@ -1493,6 +1493,7 @@ F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 F test/wal2.test 6ac39b94a284ebac6efb6be93b0cdfe73ee6083f129555e3144d8a615e9900ef F test/wal2rewrite.test 6ca6f631ffcf871240beab5f02608913fd075c6d0d31310b026c8383c65c9f9c +F test/wal2simple.test 7ea5b2bfedace701a6415d27ecac7bcd852381435b6f3d71bb90eda0db11b2b9 F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9 @@ -1521,7 +1522,6 @@ F test/walro.test 4ab7ac01b77c2f894235c699d59e3e3c7f15a160 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e -F test/waltwo2.test 6e4d36500a20ff2d19761cf0e9a5d178e83d1798feda157ebc0681e01a35e56e F test/where.test f0c325563acde44f2c4ea6ba348e9e29f7121757 F test/where2.test 478d2170637b9211f593120648858593bf2445a1 F test/where3.test 54cdeb02157acc979de41530b804ae7b09552bf1 @@ -1657,7 +1657,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9e1502e1b650217efc361732a3dfe686caa2e6352d040c73865f1faf09bf4591 8ca0fa8dfe6a66aea7fc63f15e6f704cb190aa0760a3fec2db5f6bad3861a135 -R 8c191757c669656683c7e7d83493c86d +P 7e43517861d4ecfa86766a16a2c721377b75da78771d5ba18870dcb9626a8dce +R efd9b7af2079a6949a0d309021efe433 U dan -Z b9f86bb91f1032f49d3cb4702444cf0c +Z 3fd52f62c898d986eaf1742529c99b05 diff --git a/manifest.uuid b/manifest.uuid index f045e6d6c2..1c4e653c83 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7e43517861d4ecfa86766a16a2c721377b75da78771d5ba18870dcb9626a8dce \ No newline at end of file +8932b2f1d7e6a26221ea3dea01000832b2d1eb17ac0b70ef6028f9286ae450a3 \ No newline at end of file diff --git a/test/wal2simple.test b/test/wal2simple.test new file mode 100644 index 0000000000..ea69e67b8c --- /dev/null +++ b/test/wal2simple.test @@ -0,0 +1,233 @@ +# 2017 September 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL2" mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/malloc_common.tcl +source $testdir/wal_common.tcl + +set testprefix wal2simple +ifcapable !wal {finish_test ; return } + +#------------------------------------------------------------------------- +# The following tests verify that a client can switch in and out of wal +# and wal2 mode. But that it is not possible to change directly from wal +# to wal2, or from wal2 to wal mode. +# +do_execsql_test 1.1.0 { + PRAGMA journal_mode = wal2 +} {wal2} +execsql { SELECT * FROM sqlite_master} +do_execsql_test 1.x { + PRAGMA journal_mode; + PRAGMA main.journal_mode; +} {wal2 wal2} +db close +do_test 1.1.1 { file size test.db } {1024} +do_test 1.1.2 { hexio_read test.db 18 2 } 0303 + +sqlite3 db test.db +do_execsql_test 1.2.0 { + SELECT * FROM sqlite_master; + PRAGMA journal_mode = delete; +} {delete} +db close +do_test 1.2.1 { file size test.db } {1024} +do_test 1.2.2 { hexio_read test.db 18 2 } 0101 + +sqlite3 db test.db +do_execsql_test 1.3.0 { + SELECT * FROM sqlite_master; + PRAGMA journal_mode = wal; +} {wal} +db close +do_test 1.3.1 { file size test.db } {1024} +do_test 1.3.2 { hexio_read test.db 18 2 } 0202 + +sqlite3 db test.db +do_catchsql_test 1.4.0 { + PRAGMA journal_mode = wal2; +} {1 {cannot change from wal to wal2 mode}} +do_execsql_test 1.4.1 { + PRAGMA journal_mode = wal; + PRAGMA journal_mode = delete; + PRAGMA journal_mode = wal2; + PRAGMA journal_mode = wal2; +} {wal delete wal2 wal2} +do_catchsql_test 1.4.2 { + PRAGMA journal_mode = wal; +} {1 {cannot change from wal2 to wal mode}} +db close +do_test 1.4.3 { hexio_read test.db 18 2 } 0303 + +#------------------------------------------------------------------------- +# Test that recovery in wal2 mode works. +# +forcedelete test.db test.db-wal test.db-wal2 +reset_db +do_execsql_test 2.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + PRAGMA journal_mode = wal2; + PRAGMA journal_size_limit = 5000; +} {wal2 5000} + +proc wal_hook {DB nm nFrame} { $DB eval { PRAGMA wal_checkpoint } } +db wal_hook {wal_hook db} + +for {set i 1} {$i <= 200} {incr i} { + execsql { INSERT INTO t1 VALUES(NULL, randomblob(100)) } + set res [db eval { SELECT sum(a), md5sum(b) FROM t1 }] + + do_test 2.1.$i { + foreach f [glob -nocomplain test.db2*] { forcedelete $f } + forcecopy test.db test.db2 + forcecopy test.db-wal test.db2-wal + forcecopy test.db-wal2 test.db2-wal2 + + sqlite3 db2 test.db2 + db2 eval { SELECT sum(a), md5sum(b) FROM t1 } + } $res + + db2 close +} + +#------------------------------------------------------------------------- + +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(x BLOB, y INTEGER PRIMARY KEY); + CREATE INDEX i1 ON t1(x); + PRAGMA cache_size = 5; + PRAGMA journal_mode = wal2; +} {wal2} + +do_test 3.1 { + execsql BEGIN + for {set i 1} {$i < 1000} {incr i} { + execsql { INSERT INTO t1 VALUES(randomblob(800), $i) } + } + execsql COMMIT +} {} + +do_execsql_test 3.2 { + PRAGMA integrity_check; +} {ok} + +#------------------------------------------------------------------------- +catch { db close } +foreach f [glob -nocomplain test.db*] { forcedelete $f } +reset_db +do_execsql_test 4.0 { + CREATE TABLE t1(x, y); + PRAGMA journal_mode = wal2; +} {wal2} + +do_execsql_test 4.1 { + SELECT * FROM t1; +} {} + +do_execsql_test 4.2 { + INSERT INTO t1 VALUES(1, 2); +} {} + +do_execsql_test 4.3 { + SELECT * FROM t1; +} {1 2} + +do_test 4.4 { + sqlite3 db2 test.db + execsql { SELECT * FROM t1 } db2 +} {1 2} + +do_test 4.5 { + lsort [glob test.db*] +} {test.db test.db-shm test.db-wal test.db-wal2} + +do_test 4.6 { + db close + db2 close + sqlite3 db test.db + execsql { SELECT * FROM t1 } +} {1 2} + +do_execsql_test 4.7 { + PRAGMA journal_size_limit = 4000; + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); + INSERT INTO t1 VALUES(7, 8); + INSERT INTO t1 VALUES(9, 10); + INSERT INTO t1 VALUES(11, 12); + INSERT INTO t1 VALUES(13, 14); + INSERT INTO t1 VALUES(15, 16); + INSERT INTO t1 VALUES(17, 18); + SELECT * FROM t1; +} {4000 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18} + +do_test 4.8 { + sqlite3 db2 test.db + execsql { SELECT * FROM t1 } db2 +} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18} + +do_test 4.9 { + db close + db2 close + lsort [glob test.db*] +} {test.db} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 5.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b, c); + PRAGMA journal_mode = wal2; + PRAGMA journal_size_limit = 4000; +} {wal2 4000} + +proc wal_hook {DB nm nFrame} { + $DB eval { PRAGMA wal_checkpoint } +} +db wal_hook [list wal_hook db] + + +foreach js {4000 8000 12000} { + foreach NROW [list 100 200 300 400 500 600 1000] { + do_test 5.$js.$NROW.1 { + db eval "DELETE FROM t1" + db eval "PRAGMA journal_size_limit = $js" + set nTotal 0 + for {set i 0} {$i < $NROW} {incr i} { + db eval { INSERT INTO t1 VALUES($i, $i, randomblob(abs(random()%50))) } + incr nTotal $i + } + set {} {} + } {} + + do_test 5.$js.$NROW.2 { + sqlite3 db2 test.db + db2 eval { + PRAGMA integrity_check; + SELECT count(*), sum(b) FROM t1; + } + } [list ok $NROW $nTotal] + + db2 close + } +} + + + + +finish_test diff --git a/test/waltwo2.test b/test/waltwo2.test deleted file mode 100644 index 0ab5636a06..0000000000 --- a/test/waltwo2.test +++ /dev/null @@ -1,127 +0,0 @@ -# 2017 September 19 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# This file implements regression tests for SQLite library. The -# focus of this file is testing the operation of the library in -# "PRAGMA journal_mode=WAL2" mode. -# - -set testdir [file dirname $argv0] -source $testdir/tester.tcl -source $testdir/lock_common.tcl -source $testdir/malloc_common.tcl -source $testdir/wal_common.tcl - -set testprefix walsimple -ifcapable !wal {finish_test ; return } - -db close -foreach f [glob -nocomplain test.db*] { forcedelete $f } -sqlite3 db test.db - -do_execsql_test 1.0 { - CREATE TABLE t1(x, y); - PRAGMA journal_mode = wal2; -} {wal2} - -do_execsql_test 1.1 { - SELECT * FROM t1; -} {} - -do_execsql_test 1.2 { - INSERT INTO t1 VALUES(1, 2); -} {} - -do_execsql_test 1.3 { - SELECT * FROM t1; -} {1 2} - -do_test 1.4 { - sqlite3 db2 test.db - execsql { SELECT * FROM t1 } db2 -} {1 2} - -do_test 1.5 { - lsort [glob test.db*] -} {test.db test.db-shm test.db-wal test.db-wal2} - -do_test 1.6 { - db close - db2 close - sqlite3 db test.db - execsql { SELECT * FROM t1 } -} {1 2} - -do_execsql_test 1.7 { - PRAGMA journal_size_limit = 4000; - INSERT INTO t1 VALUES(3, 4); - INSERT INTO t1 VALUES(5, 6); - INSERT INTO t1 VALUES(7, 8); - INSERT INTO t1 VALUES(9, 10); - INSERT INTO t1 VALUES(11, 12); - INSERT INTO t1 VALUES(13, 14); - INSERT INTO t1 VALUES(15, 16); - INSERT INTO t1 VALUES(17, 18); - SELECT * FROM t1; -} {4000 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18} - -do_test 1.8 { - sqlite3 db2 test.db - execsql { SELECT * FROM t1 } db2 -} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18} - -do_test 1.9 { - db close - db2 close - lsort [glob test.db*] -} {test.db} - -#------------------------------------------------------------------------- -reset_db -do_execsql_test 2.0 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); - CREATE INDEX i1 ON t1(b, c); - PRAGMA journal_mode = wal2; - PRAGMA journal_size_limit = 4000; -} {wal2 4000} - -proc wal_hook {DB nm nFrame} { - $DB eval { PRAGMA wal_checkpoint } -} -db wal_hook [list wal_hook db] - - -foreach js {4000 8000 12000} { - foreach NROW [list 100 200 300 400 500 600 1000] { - do_test 2.$js.$NROW.1 { - db eval "DELETE FROM t1" - db eval "PRAGMA journal_size_limit = $js" - set nTotal 0 - for {set i 0} {$i < $NROW} {incr i} { - db eval { INSERT INTO t1 VALUES($i, $i, randomblob(abs(random()%50))) } - incr nTotal $i - } - set {} {} - } {} - - do_test 2.$js.$NROW.2 { - sqlite3 db2 test.db - db2 eval { - PRAGMA integrity_check; - SELECT count(*), sum(b) FROM t1; - } - } [list ok $NROW $nTotal] - - db2 close - } -} - -finish_test - From 50182fa846d66e413e303e2545885ac12667b598 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 7 Oct 2017 19:55:37 +0000 Subject: [PATCH 078/179] Ignore the *-wal2 file if the *-wal file is zero bytes in size. FossilOrigin-Name: f7360fad51f224f347bb7d263eb89056b27461c278309e00e575a0e8898c9f40 --- manifest | 14 +++++------ manifest.uuid | 2 +- src/wal.c | 20 +++++++++++++++ test/wal2simple.test | 58 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 86 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 5356b92b47..aa48e86c68 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Check\sin\stest\sfile\swal2simple.test. -D 2017-10-07T13:37:04.814 +C Ignore\sthe\s*-wal2\sfile\sif\sthe\s*-wal\sfile\sis\szero\sbytes\sin\ssize. +D 2017-10-07T19:55:37.322 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f @@ -537,7 +537,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 287feccc5c6e886a5d747f5555382280e85a24904d0378940ef3869eb4273571 +F src/wal.c 11314f64edbb7613e80290830a7379ff083b511bd1ee97846518c3102ebf5c4f F src/wal.h b6063e6be1b03389372f3f32240e99b8ab92c32cdd05aa0e31b30a21e4e41654 F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6 F src/where.c 049522adcf5426f1a8c3ed07be15e1ffa3266afd34e8e7bee64b63e2fbfad0b5 @@ -1493,7 +1493,7 @@ F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 F test/wal2.test 6ac39b94a284ebac6efb6be93b0cdfe73ee6083f129555e3144d8a615e9900ef F test/wal2rewrite.test 6ca6f631ffcf871240beab5f02608913fd075c6d0d31310b026c8383c65c9f9c -F test/wal2simple.test 7ea5b2bfedace701a6415d27ecac7bcd852381435b6f3d71bb90eda0db11b2b9 +F test/wal2simple.test 8719413446ca97ca88507c5b79f139b631faa5b4e177b7424008b26fc153da30 F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9 @@ -1657,7 +1657,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7e43517861d4ecfa86766a16a2c721377b75da78771d5ba18870dcb9626a8dce -R efd9b7af2079a6949a0d309021efe433 +P 8932b2f1d7e6a26221ea3dea01000832b2d1eb17ac0b70ef6028f9286ae450a3 +R c46e3ca31dc60584535891a8280b1e6f U dan -Z 3fd52f62c898d986eaf1742529c99b05 +Z fe4e7b40b37235ec7703f9add7af6779 diff --git a/manifest.uuid b/manifest.uuid index 1c4e653c83..bd32a9a8c9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8932b2f1d7e6a26221ea3dea01000832b2d1eb17ac0b70ef6028f9286ae450a3 \ No newline at end of file +f7360fad51f224f347bb7d263eb89056b27461c278309e00e575a0e8898c9f40 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 2cc21039da..277f674bbb 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1375,6 +1375,21 @@ static int walOpenWal2(Wal *pWal){ return rc; } +static int walTruncateWal2(Wal *pWal){ + int bIs; + int rc; + assert( !isOpen(pWal->apWalFd[1]) ); + rc = sqlite3OsAccess(pWal->pVfs, pWal->zWalName2, SQLITE_ACCESS_EXISTS, &bIs); + if( rc==SQLITE_OK && bIs ){ + rc = walOpenWal2(pWal); + if( rc==SQLITE_OK ){ + rc = sqlite3OsTruncate(pWal->apWalFd[1], 0); + sqlite3OsClose(pWal->apWalFd[1]); + } + } + return rc; +} + /* ** Recover the wal-index by reading the write-ahead log file. ** @@ -1434,6 +1449,9 @@ static int walIndexRecover(Wal *pWal){ || pWal->hdr.iVersion==WAL_VERSION1 || pWal->hdr.iVersion==WAL_VERSION2 ); + if( rc==SQLITE_OK && bZero ){ + rc = walTruncateWal2(pWal); + } if( rc==SQLITE_OK && pWal->hdr.iVersion!=WAL_VERSION1 ){ int bOpen = 1; sqlite3_vfs *pVfs = pWal->pVfs; @@ -1481,6 +1499,8 @@ static int walIndexRecover(Wal *pWal){ pWal->hdr = hdr; }else{ walidxSetFile(&pWal->hdr, 1); + walidxSetMxFrame(&pWal->hdr, 1, pWal->hdr.mxFrame); + walidxSetMxFrame(&pWal->hdr, 0, 0); } pWal->hdr.iVersion = WAL_VERSION2; }else{ diff --git a/test/wal2simple.test b/test/wal2simple.test index ea69e67b8c..2498c5500f 100644 --- a/test/wal2simple.test +++ b/test/wal2simple.test @@ -228,6 +228,64 @@ foreach js {4000 8000 12000} { } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 6.0 { + CREATE TABLE tx(x); + PRAGMA journal_mode = wal2; + PRAGMA journal_size_limit = 3500; +} {wal2 3500} + +do_test 6.1 { + for {set i 0} {$i < 10} {incr i} { + execsql "CREATE TABLE t$i (x);" + } +} {} + +puts "[file size test.db-wal] [file size test.db-wal2]" + +do_test 6.2.1 { + foreach f [glob -nocomplain test.db2*] { forcedelete $f } + forcecopy test.db-wal2 test.db2-wal2 + sqlite3 db2 test.db2 + db2 eval { SELECT * FROM sqlite_master } +} {} +do_test 6.2.2 { + db2 eval { + PRAGMA journal_mode = wal2; + SELECT * FROM sqlite_master; + } +} {wal2} + +do_test 6.3.1 { + db2 close + foreach f [glob -nocomplain test.db2*] { forcedelete $f } + forcecopy test.db-wal2 test.db2-wal2 + forcecopy test.db test.db2 + sqlite3 db2 test.db2 + db2 eval { SELECT * FROM sqlite_master } +} {table tx tx 2 {CREATE TABLE tx(x)}} +do_test 6.3.2 { + db2 eval { + PRAGMA journal_mode = wal2; + SELECT * FROM sqlite_master; + } +} {wal2 table tx tx 2 {CREATE TABLE tx(x)}} +do_test 6.4.1 { + db2 close + foreach f [glob -nocomplain test.db2*] { forcedelete $f } + forcecopy test.db-wal2 test.db2-wal2 + forcecopy test.db-wal test.db2-wal + sqlite3 db2 test.db2 + db2 eval { SELECT * FROM sqlite_master } +} {} +do_test 6.4.2 { + db2 eval { + PRAGMA journal_mode = wal2; + SELECT * FROM sqlite_master; + } +} {wal2} +db2 close finish_test From 40927fd61e4db1149e31a65d4ca05d64bb6212dc Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 9 Oct 2017 19:49:08 +0000 Subject: [PATCH 079/179] Add a header comment to wal.c describing the differences between wal and wal2 mode. FossilOrigin-Name: 9c80cd202f1c966929b279e18f19c663912686fcf92f03b85a02b9c7e55a0fc6 --- manifest | 12 ++-- manifest.uuid | 2 +- src/wal.c | 168 +++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 173 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index aa48e86c68..0f5352281d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ignore\sthe\s*-wal2\sfile\sif\sthe\s*-wal\sfile\sis\szero\sbytes\sin\ssize. -D 2017-10-07T19:55:37.322 +C Add\sa\sheader\scomment\sto\swal.c\sdescribing\sthe\sdifferences\sbetween\swal\sand\swal2\nmode. +D 2017-10-09T19:49:08.605 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f @@ -537,7 +537,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 11314f64edbb7613e80290830a7379ff083b511bd1ee97846518c3102ebf5c4f +F src/wal.c c025455c9d6cf48ca55bd894be4a37a160565de9c845510a048fb9113761b2f4 F src/wal.h b6063e6be1b03389372f3f32240e99b8ab92c32cdd05aa0e31b30a21e4e41654 F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6 F src/where.c 049522adcf5426f1a8c3ed07be15e1ffa3266afd34e8e7bee64b63e2fbfad0b5 @@ -1657,7 +1657,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8932b2f1d7e6a26221ea3dea01000832b2d1eb17ac0b70ef6028f9286ae450a3 -R c46e3ca31dc60584535891a8280b1e6f +P f7360fad51f224f347bb7d263eb89056b27461c278309e00e575a0e8898c9f40 +R 3eebf17818b82033242954f80ad24b93 U dan -Z fe4e7b40b37235ec7703f9add7af6779 +Z fe65619fadae77c6923d24a16aa4aa37 diff --git a/manifest.uuid b/manifest.uuid index bd32a9a8c9..f96be1474c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f7360fad51f224f347bb7d263eb89056b27461c278309e00e575a0e8898c9f40 \ No newline at end of file +9c80cd202f1c966929b279e18f19c663912686fcf92f03b85a02b9c7e55a0fc6 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 277f674bbb..836e876c5c 100644 --- a/src/wal.c +++ b/src/wal.c @@ -101,7 +101,7 @@ ** ** To read a page from the database (call it page number P), a reader ** first checks the WAL to see if it contains page P. If so, then the -** last valid instance of page P that is a followed by a commit frame +** last valid instance of page P that is followed by a commit frame ** or is a commit frame itself becomes the value read. If the WAL ** contains no copies of page P that are valid and which are a commit ** frame or are followed by a commit frame, then page P is read from @@ -229,7 +229,7 @@ ** and to the wal-index) might be using a different value K1, where K1>K0. ** Both readers can use the same hash table and mapping section to get ** the correct result. There may be entries in the hash table with -** K>K0 but to the first reader, those entries will appear to be unused +** K>K0, but to the first reader those entries will appear to be unused ** slots in the hash table and so the first reader will get an answer as ** if no values greater than K0 had ever been inserted into the hash table ** in the first place - which is what reader one wants. Meanwhile, the @@ -240,6 +240,166 @@ ** that correspond to frames greater than the new K value are removed ** from the hash table at this point. */ + +/* +** WAL2 NOTES +** +** This file also contains the implementation of "wal2" mode - activated +** using "PRAGMA journal_mode = wal2". Wal2 mode is very similar to wal +** mode, except that it uses two wal files instead of one. Under some +** circumstances, wal2 mode provides more concurrency than legacy wal +** mode. +** +** THE PROBLEM WAL2 SOLVES: +** +** In legacy wal mode, if a writer wishes to write to the database while +** a checkpoint is ongoing, it may append frames to the existing wal file. +** This means that after the checkpoint has finished, the wal file consists +** of a large block of checkpointed frames, followed by a block of +** uncheckpointed frames. In a deployment that features a high volume of +** write traffic, this may mean that the wal file is never completely +** checkpointed. And so grows indefinitely. +** +** An alternative is to use "PRAGMA wal_checkpoint=RESTART" or similar to +** force a complete checkpoint of the wal file. But this must: +** +** 1) Wait on all existing readers to finish, +** 2) Wait on any existing writer, and then block all new writers, +** 3) Do the checkpoint, +** 4) Wait on any new readers that started during steps 2 and 3. Writers +** are still blocked during this step. +** +** This means that in order to avoid the wal file growing indefinitely +** in a busy system, writers must periodically pause to allow a checkpoint +** to complete. In a system with long running readers, such pauses may be +** for a non-trivial amount of time. +** +** OVERVIEW OF SOLUTION +** +** Wal2 mode uses two wal files. After writers have grown the first wal +** file to a pre-configured size, they begin appending transactions to +** the second wal file. Once all existing readers are reading snapshots +** new enough to include the entire first wal file, a checkpointer can +** checkpoint it. +** +** Meanwhile, writers are writing transactions to the second wal file. +** Once that wal file has grown larger than the pre-configured size, each +** new writer checks if: +** +** * the first wal file has been checkpointed, and if so, if +** * there are no readers still reading from the first wal file (once +** it has been checkpointed, new readers read only from the second +** wal file). +** +** If both these conditions are true, the writer may switch back to the +** first wal file. Eventually, a checkpointer can checkpoint the second +** wal file, and so on. +** +** The wal file that writers are currently appending to (the one they +** don't have to check the above two criteria before writing to) is called +** the "current" wal file. +** +** The first wal file takes the same name as the wal file in legacy wal +** mode systems - "-wal". The second is named "-wal2". +** +** WAL FILE FORMAT +** +** The file format used for each wal file in wal2 mode is the same as for +** legacy wal mode. Except, the file format field is set to 3021000 +** instead of 3007000. +** +** WAL-INDEX FORMAT +** +** The wal-index format is also very similar. Even though there are two +** wal files, there is still a single wal-index shared-memory area (*-shm +** file with the default unix or win32 VFS). The wal-index header is the +** same size, with the following exceptions it has the same format: +** +** * The version field is set to 3021000 instead of 3007000. +** +** * An unused 32-bit field in the legacy wal-index header is +** now used to store (a) a single bit indicating which of the +** two wal files writers should append to and (b) the number +** of frames in the second wal file (31 bits). +** +** The first hash table in the wal-index contains entries corresponding +** to the first HASHTABLE_NPAGE_ONE frames stored in the first wal file. +** The second hash table in the wal-index contains entries indexing the +** first HASHTABLE_NPAGE frames in the second wal file. The third hash +** table contains the next HASHTABLE_NPAGE frames in the first wal file, +** and so on. +** +** LOCKS +** +** Read-locks are simpler than for legacy wal mode. There are no locking +** slots that contain frame numbers. Instead, there are four distinct +** combinations of read locks a reader may hold: +** +** WAL_LOCK_PART1: "part" lock on first wal, none of second. +** WAL_LOCK_PART1_FULL2: "part" lock on first wal, "full" of second. +** WAL_LOCK_PART2: no lock on first wal, "part" lock on second. +** WAL_LOCK_PART2_FULL1: "full" lock on first wal, "part" lock on second. +** +** When a reader reads the wal-index header as part of opening a read +** transaction, it takes a "part" lock on the current wal file. "Part" +** because the wal file may grow while the read transaction is active, in +** which case the reader would be reading only part of the wal file. +** A part lock prevents a checkpointer from checkpointing the wal file +** on which it is held. +** +** If there is data in the non-current wal file that has not been +** checkpointed, the reader takes a "full" lock on that wal file. A +** "full" lock indicates that the reader is using the entire wal file. +** A full lock prevents a writer from overwriting the wal file on which +** it is held, but does not prevent a checkpointer from checkpointing +** it. +** +** There is still a single WRITER and a single CHECKPOINTER lock. The +** recovery procedure still takes the same exclusive lock on the entire +** range of SQLITE_SHM_NLOCK shm-locks. This works because the read-locks +** above use four of the six read-locking slots used by legacy wal mode. +** See the header comment for function walLockReader() for details. +** +** STARTUP/RECOVERY +** +** The read and write version fields of the database header in a wal2 +** database are set to 0x03, instead of 0x02 as in legacy wal mode. +** +** The wal file format used in wal2 mode is the same as the format used +** in legacy wal mode. However, in order to support recovery, there are two +** differences in the way wal file header fields are populated, as follows: +** +** * When the first wal file is first created, the "nCkpt" field in +** the wal file header is set to 0. Thereafter, each time the writer +** switches wal file, it sets the nCkpt field in the new wal file +** header to ((nCkpt0 + 1) & 0x0F), where nCkpt0 is the value in +** the previous wal file header. This means that the first wal file +** always has an even value in the nCkpt field, and the second wal +** file always has an odd value. +** +** * When a writer switches wal file, it sets the salt values in the +** new wal file to a copy of the checksum for the final frame in +** the previous wal file. +** +** Recovery proceeds as follows: +** +** 1. Each wal file is recovered separately. Except, if the first wal +** file does not exist or is zero bytes in size, the second wal file +** is truncated to zero bytes before it is "recovered". +** +** 2. If both wal files contain valid headers, then the nCkpt fields +** are compared to see which of the two wal files is older. If the +** salt keys in the second wal file match the final frame checksum +** in the older wal file, then both wal files are used. Otherwise, +** the newer wal file is ignored. +** +** 3. Or, if only one or neither of the wal files has a valid header, +** then only a single or no wal files are recovered into the +** reconstructed wal-index. +** +** Refer to header comments for walIndexRecover() for further details. +*/ + #ifndef SQLITE_OMIT_WAL #include "wal.h" @@ -1491,6 +1651,10 @@ static int walIndexRecover(Wal *pWal){ ){ SWAP(WalIndexHdr, pWal->hdr, hdr); walidxSetMxFrame(&pWal->hdr, 1, hdr.mxFrame); + }else{ + walidxSetFile(&pWal->hdr, 1); + walidxSetMxFrame(&pWal->hdr, 1, pWal->hdr.mxFrame); + walidxSetMxFrame(&pWal->hdr, 0, 0); } }else From 6cbf8e173f69cda68209f879661803b37606fa53 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 10 Oct 2017 20:11:10 +0000 Subject: [PATCH 080/179] Add new extension "bgckpt" in ext/misc/bgckpt.c. For experimenting with running wal2 mode checkpoints in a background thread. FossilOrigin-Name: 63955442304052f5adddd05ccaeebe87ddc5e25af695702793518f015b4f0a87 --- ext/misc/bgckpt.c | 238 ++++++++++++++++++++++++++++++++++++++++++++++ main.mk | 1 + manifest | 15 +-- manifest.uuid | 2 +- src/tclsqlite.c | 2 + 5 files changed, 250 insertions(+), 8 deletions(-) create mode 100644 ext/misc/bgckpt.c diff --git a/ext/misc/bgckpt.c b/ext/misc/bgckpt.c new file mode 100644 index 0000000000..ca8f3edf01 --- /dev/null +++ b/ext/misc/bgckpt.c @@ -0,0 +1,238 @@ +/* +** 2017-10-11 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ + +#if !defined(SQLITE_TEST) || defined(SQLITE_OS_UNIX) + +#include "sqlite3.h" +#include +#include + +/* +** API declarations. +*/ +typedef struct Checkpointer Checkpointer; +int sqlite3_bgckpt_create(const char *zFilename, Checkpointer **pp); +int sqlite3_bgckpt_checkpoint(Checkpointer *p, int bBlock); +void sqlite3_bgckpt_destroy(Checkpointer *p); + + +struct Checkpointer { + sqlite3 *db; /* Database handle */ + + pthread_t thread; /* Background thread */ + pthread_mutex_t mutex; + pthread_cond_t cond; + + int rc; /* Error from "PRAGMA wal_checkpoint" */ + int bCkpt; /* True if checkpoint requested */ + int bExit; /* True if exit requested */ +}; + +static void *bgckptThreadMain(void *pCtx){ + int rc = SQLITE_OK; + Checkpointer *p = (Checkpointer*)pCtx; + + while( rc==SQLITE_OK ){ + int bExit; + + pthread_mutex_lock(&p->mutex); + if( p->bCkpt==0 && p->bExit==0 ){ + pthread_cond_wait(&p->cond, &p->mutex); + } + p->bCkpt = 0; + bExit = p->bExit; + pthread_mutex_unlock(&p->mutex); + + if( bExit ) break; + rc = sqlite3_exec(p->db, "PRAGMA wal_checkpoint", 0, 0, 0); + if( rc==SQLITE_BUSY ){ + rc = SQLITE_OK; + } + } + + pthread_mutex_lock(&p->mutex); + p->rc = rc; + pthread_mutex_unlock(&p->mutex); + return 0; +} + +void sqlite3_bgckpt_destroy(Checkpointer *p){ + if( p ){ + void *ret = 0; + + /* Signal the background thread to exit */ + pthread_mutex_lock(&p->mutex); + p->bExit = 1; + pthread_cond_broadcast(&p->cond); + pthread_mutex_unlock(&p->mutex); + + pthread_join(p->thread, &ret); + sqlite3_close(p->db); + sqlite3_free(p); + } +} + + +int sqlite3_bgckpt_create(const char *zFilename, Checkpointer **pp){ + Checkpointer *pNew = 0; + int rc; + + pNew = (Checkpointer*)sqlite3_malloc(sizeof(Checkpointer)); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pNew, 0, sizeof(Checkpointer)); + rc = sqlite3_open(zFilename, &pNew->db); + } + + if( rc==SQLITE_OK ){ + pthread_mutex_init(&pNew->mutex, 0); + pthread_cond_init(&pNew->cond, 0); + pthread_create(&pNew->thread, 0, bgckptThreadMain, (void*)pNew); + } + + if( rc!=SQLITE_OK ){ + sqlite3_bgckpt_destroy(pNew); + pNew = 0; + } + *pp = pNew; + return rc; +} + +int sqlite3_bgckpt_checkpoint(Checkpointer *p, int bBlock){ + int rc; + pthread_mutex_lock(&p->mutex); + rc = p->rc; + if( rc==SQLITE_OK ){ + p->bCkpt = 1; + pthread_cond_broadcast(&p->cond); + } + pthread_mutex_unlock(&p->mutex); + return rc; +} + +#ifdef SQLITE_TEST + +#if defined(INCLUDE_SQLITE_TCL_H) +# include "sqlite_tcl.h" +#else +# include "tcl.h" +# ifndef SQLITE_TCLAPI +# define SQLITE_TCLAPI +# endif +#endif + +const char *sqlite3ErrName(int rc); + +static void SQLITE_TCLAPI bgckpt_del(void * clientData){ + Checkpointer *pCkpt = (Checkpointer*)clientData; + sqlite3_bgckpt_destroy(pCkpt); +} + +/* +** Tclcmd: $ckpt SUBCMD ... +*/ +static int SQLITE_TCLAPI bgckpt_obj_cmd( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + Checkpointer *pCkpt = (Checkpointer*)clientData; + const char *aCmd[] = { "checkpoint", "destroy", 0 }; + int iCmd; + + if( objc<2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "SUBCMD ..."); + return TCL_ERROR; + } + + if( Tcl_GetIndexFromObj(interp, objv[1], aCmd, "sub-command", 0, &iCmd) ){ + return TCL_ERROR; + } + + switch( iCmd ){ + case 0: { + int rc; + int bBlock = 0; + + if( objc>3 ){ + Tcl_WrongNumArgs(interp, 2, objv, "?BLOCKING?"); + return TCL_ERROR; + } + if( objc==3 && Tcl_GetBooleanFromObj(interp, objv[2], &bBlock) ){ + return TCL_ERROR; + } + + rc = sqlite3_bgckpt_checkpoint(pCkpt, bBlock); + if( rc!=SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + return TCL_ERROR; + } + break; + } + + case 1: { + Tcl_DeleteCommand(interp, Tcl_GetString(objv[0])); + break; + } + } + + return TCL_OK; +} + +/* +** Tclcmd: bgckpt CMDNAME FILENAME +*/ +static int SQLITE_TCLAPI bgckpt_cmd( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + const char *zCmd; + const char *zFilename; + int rc; + Checkpointer *pCkpt; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "CMDNAME FILENAME"); + return TCL_ERROR; + } + zCmd = Tcl_GetString(objv[1]); + zFilename = Tcl_GetString(objv[2]); + + rc = sqlite3_bgckpt_create(zFilename, &pCkpt); + if( rc!=SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + return TCL_ERROR; + } + + Tcl_CreateObjCommand(interp, zCmd, bgckpt_obj_cmd, (void*)pCkpt, bgckpt_del); + Tcl_SetObjResult(interp, objv[1]); + return TCL_OK; +} + +int Bgckpt_Init(Tcl_Interp *interp){ + Tcl_CreateObjCommand(interp, "bgckpt", bgckpt_cmd, 0, 0); + return TCL_OK; +} +#endif /* SQLITE_TEST */ + +#else +int Bgckpt_Init(Tcl_Interp *interp){ + return TCL_OK; +} +#endif + diff --git a/main.mk b/main.mk index 7da6db15f1..e34c88181b 100644 --- a/main.mk +++ b/main.mk @@ -327,6 +327,7 @@ TESTSRC = \ # TESTSRC += \ $(TOP)/ext/misc/amatch.c \ + $(TOP)/ext/misc/bgckpt.c \ $(TOP)/ext/misc/carray.c \ $(TOP)/ext/misc/closure.c \ $(TOP)/ext/misc/csv.c \ diff --git a/manifest b/manifest index 8bc427df49..709f88d499 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\swith\sthis\sbranch. -D 2017-10-09T19:50:09.728 +C Add\snew\sextension\s"bgckpt"\sin\sext/misc/bgckpt.c.\sFor\sexperimenting\swith\nrunning\swal2\smode\scheckpoints\sin\sa\sbackground\sthread. +D 2017-10-10T20:11:10.809 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f @@ -258,6 +258,7 @@ F ext/lsm1/test/lsm1_simple.test ca949efefa102f4644231dcd9291d8cda7699a4ce1006b2 F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb +F ext/misc/bgckpt.c 1e0178c1d9f44d44f45c731ebff45854194ca59a41d33f078f10a3a714bf4532 F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704 F ext/misc/completion.c 52c3f01523e3e387eb321b4739a89d1fe47cbe6025aa1f2d8d3685e9e365df0f @@ -382,7 +383,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk d0145f02deb67d65c4822225847cba112c237cdb62f4905eeb4b648e82bfc222 +F main.mk 8dcd78e5977c6e220ec0ab9a00aa2469ca38380d1553cb3eff18469a28e04654 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -468,7 +469,7 @@ F src/sqliteInt.h c07bc88eca1f59ce73e1f486187d0df4effe67c4579e112dfdd91c159e5c05 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 -F src/tclsqlite.c 487951d81f9704800fd9f0ffdaa2f935a83ccb6be3575c2c4ef83e4789b4c828 +F src/tclsqlite.c 37be19e8ea1b4ec1e632d86272ada77bbb93f20b033df258ad3ed63dc4d40fa4 F src/test1.c 8ef15f7a357f85dfc41c6c748ce9c947b4f676e01bb5ae6a45bee4923dff8b51 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b @@ -1658,7 +1659,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9c80cd202f1c966929b279e18f19c663912686fcf92f03b85a02b9c7e55a0fc6 5594a121bf132a98d0ecb4cf86d9f2681925c9416206096bf11c9370a5dae22f -R 23a883a99c179b06ab669012d3314db6 +P d218d815f89cb1368fdb5e3f774b7adaaf02560a367ba0f3e54987e08dd6241a +R 7d8085c250f90df872ed6ab275626e25 U dan -Z c8f907ef387c1600d03d31327cf9ed02 +Z bca32e00ecb94225e888df6bb3b83e85 diff --git a/manifest.uuid b/manifest.uuid index 7c4ede0c0c..91bc8d5bae 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d218d815f89cb1368fdb5e3f774b7adaaf02560a367ba0f3e54987e08dd6241a \ No newline at end of file +63955442304052f5adddd05ccaeebe87ddc5e25af695702793518f015b4f0a87 \ No newline at end of file diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 1b9f91405e..29fe8f7953 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -4183,6 +4183,7 @@ static void init_all(Tcl_Interp *interp){ extern int Fts5tcl_Init(Tcl_Interp *); extern int SqliteRbu_Init(Tcl_Interp*); extern int Sqlitetesttcl_Init(Tcl_Interp*); + extern int Bgckpt_Init(Tcl_Interp*); #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) extern int Sqlitetestfts3_Init(Tcl_Interp *interp); #endif @@ -4231,6 +4232,7 @@ static void init_all(Tcl_Interp *interp){ Fts5tcl_Init(interp); SqliteRbu_Init(interp); Sqlitetesttcl_Init(interp); + Bgckpt_Init(interp); #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) Sqlitetestfts3_Init(interp); From 56391f2769aa384088175b24d28f37a1526f2390 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Nov 2017 18:48:54 +0000 Subject: [PATCH 081/179] The "PRAGMA noop_update" command now requires SQLITE_ENABLE_NOOP_UPDATE and no longer requires SQLITE_DEBUG. FossilOrigin-Name: 81baf67c4493468e4feb2f4990bf82d59804ce4f3149252c0e1e8c43f90d6bc1 --- manifest | 20 ++++++++------------ manifest.uuid | 2 +- src/pragma.h | 2 +- src/update.c | 2 +- tool/mkpragmatab.tcl | 2 +- 5 files changed, 12 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index aba94857ed..ff67f1b298 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\shighly-experimental\s"PRAGMA\snoop_update=TRUE"\scommand. -D 2017-09-12T20:09:31.560 +C The\s"PRAGMA\snoop_update"\scommand\snow\srequires\sSQLITE_ENABLE_NOOP_UPDATE\nand\sno\slonger\srequires\sSQLITE_DEBUG. +D 2017-11-01T18:48:54.596 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016 @@ -451,7 +451,7 @@ F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 521bb9610d38ef17a3cc9b5ddafd4546c2ea67fa3d0e464823d73c2a28d50e11 F src/pcache1.c 0b793738b5dddaf0a645784835c6b5557b1ecfaee339af9c26810c6ecdb273aa F src/pragma.c cd6aeda3587be6c5c08f9b2d45eae6068666a03c9d077c8c43cdb85fb0aa70f2 -F src/pragma.h 70cb22e66adabf21bac20e2894184ece986a67a3252506220f3b62633c87dceb +F src/pragma.h ce41efb7d4cdafca499839f29014d9b1d9534c8f503eeceb88310920c62d6097 F src/prepare.c 3cbb99757d7295997674972f9dd2331c5c544368854ca08954c9beb1e9b6145a F src/printf.c 8757834f1b54dae512fb25eb1acc8e94a0d15dd2290b58f2563f65973265adb2 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 @@ -521,7 +521,7 @@ F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 1003d6d90c6783206c711f0a9397656fa5b055209f4d092caa43bb3bf5215db5 F src/treeview.c 2ee4a5dada213d5ab08a742af5c876cee6f1aaae65f10a61923f3fb63846afef F src/trigger.c 48e0f7ed6749ce4d50a695e09e20ce9cf84ecabf2691852c965a51e0b620eccc -F src/update.c 2bb1d048a1a56e1f9238d68e8e476299a5873c4d4d9e0e9c4ab542d8b25f341d +F src/update.c d36ee12b6cde68543fa35f6b88475a78e787c7237ed0a79c786364e8ef680480 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 F src/util.c fc081ec6f63448dcd80d3dfad35baecfa104823254a815b081a4d9fe76e1db23 F src/vacuum.c 07eec96f821c7dcbdca2fadffc6e38ea2c24bf409fcb15fe9fb3ac444d632dfe @@ -1594,7 +1594,7 @@ F tool/mkmsvcmin.tcl cbd93f1cfa3a0a9ae56fc958510aa3fc3ac65e29cb111716199e3d0e66e F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c F tool/mkopcodeh.tcl 4ee2a30ccbd900dc4d5cdb61bdab87cd2166cd2affcc78c9cc0b8d22a65b2eee F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e -F tool/mkpragmatab.tcl e9a725395dc4419047d242f4ebd4716dd8c682e234b6553a4bb891fc1f12174c +F tool/mkpragmatab.tcl 0d67312ad1a6db29e85215c87e3f5c0cd3a9bf9e32784bf3a02e5ebaf1faadb0 F tool/mkshellc.tcl 69c38ecd7b74b2b0799a35ce20e1e3998e504d8c99c100ca4b98ae9d8f6279bc F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb @@ -1657,11 +1657,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4256072399f44f48ed0856aa8112226af6feaf8676923612bde6cea239ebf920 -Q +de2e371757d2031cefc4bbae29d746a768c126a6c8443bb0f9ddebee7e69240b -R 2413782f3f576a461bb90e166b6f6e32 -T *branch * begin-concurrent-pnu -T *sym-begin-concurrent-pnu * -T -sym-begin-concurrent * +P afe45271b9c0cd379cf0beb94657e2396068c4a18f84003c4c48297760fd83ee +R 13a4e608c33e96d8f32961fb092111d1 U drh -Z 105db701de986bc4ba15a35d9febe596 +Z 22f49919fb59529cdea1770ba1acc6e3 diff --git a/manifest.uuid b/manifest.uuid index c3c05f6cd6..7e93f679a5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -afe45271b9c0cd379cf0beb94657e2396068c4a18f84003c4c48297760fd83ee \ No newline at end of file +81baf67c4493468e4feb2f4990bf82d59804ce4f3149252c0e1e8c43f90d6bc1 \ No newline at end of file diff --git a/src/pragma.h b/src/pragma.h index 80801945cf..c4f61734f8 100644 --- a/src/pragma.h +++ b/src/pragma.h @@ -442,7 +442,7 @@ static const PragmaName aPragmaName[] = { #endif #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) -#if defined(SQLITE_DEBUG) +#if defined(SQLITE_ENABLE_NOOP_UPDATE) {/* zName: */ "noop_update", /* ePragTyp: */ PragTyp_FLAG, /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, diff --git a/src/update.c b/src/update.c index 912b7e646b..86872cd3c0 100644 --- a/src/update.c +++ b/src/update.c @@ -223,7 +223,7 @@ void sqlite3Update( */ chngRowid = chngPk = 0; for(i=0; inExpr; i++){ -#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if defined(SQLITE_ENABLE_NOOP_UPDATE) && !defined(SQLITE_OMIT_FLAG_PRAGMAS) if( db->flags & SQLITE_NoopUpdate ){ Token x; sqlite3ExprDelete(db, pChanges->a[i].pExpr); diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index b790398476..4d4b959aa1 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -116,7 +116,7 @@ set pragma_def { TYPE: FLAG ARG: SQLITE_NoopUpdate IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) - IF: defined(SQLITE_DEBUG) + IF: defined(SQLITE_ENABLE_NOOP_UPDATE) NAME: ignore_check_constraints TYPE: FLAG From 4d13729861259c527a56cd5e7e79ef51c41a4b0d Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Nov 2017 19:30:41 +0000 Subject: [PATCH 082/179] Fix the SQLITE_NoopUpdate #define so that it occurs under the correct conditions. FossilOrigin-Name: bdf791f9f7f7ab4b1871e1d3a0d5edcad46fc67d7336ae7f3f7b20e69801be8e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqliteInt.h | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ff67f1b298..28ecb53069 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\s"PRAGMA\snoop_update"\scommand\snow\srequires\sSQLITE_ENABLE_NOOP_UPDATE\nand\sno\slonger\srequires\sSQLITE_DEBUG. -D 2017-11-01T18:48:54.596 +C Fix\sthe\sSQLITE_NoopUpdate\s#define\sso\sthat\sit\soccurs\sunder\sthe\scorrect\nconditions. +D 2017-11-01T19:30:41.289 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016 @@ -463,7 +463,7 @@ F src/shell.c.in b5725acacba95ccefa57b6d068f710e29ba8239c3aa704628a1902a1f729c17 F src/sqlite.h.in 803d2c969bccaf78ef087269c73f1f00f8870c122b3514414b8c47c4fde73e82 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47 -F src/sqliteInt.h dc8a03250ca4887630a048ed135e50c47f75ef30a0859256420d072e6b98caf4 +F src/sqliteInt.h 5bea474f3e9e2446bb80d3f5e632fb33cad6332f822fd1f207c4fbfee63bc39c F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1657,7 +1657,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P afe45271b9c0cd379cf0beb94657e2396068c4a18f84003c4c48297760fd83ee -R 13a4e608c33e96d8f32961fb092111d1 +P 81baf67c4493468e4feb2f4990bf82d59804ce4f3149252c0e1e8c43f90d6bc1 +R 59abeaa6daabac9ab7e74c5755dea792 U drh -Z 22f49919fb59529cdea1770ba1acc6e3 +Z d4cba171c15be720c948ceaa78bd9661 diff --git a/manifest.uuid b/manifest.uuid index 7e93f679a5..97b4af9f46 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -81baf67c4493468e4feb2f4990bf82d59804ce4f3149252c0e1e8c43f90d6bc1 \ No newline at end of file +bdf791f9f7f7ab4b1871e1d3a0d5edcad46fc67d7336ae7f3f7b20e69801be8e \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index c6295c3d33..dfc3cb6564 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1488,9 +1488,9 @@ struct sqlite3 { #define SQLITE_CellSizeCk 0x00200000 /* Check btree cell sizes on load */ #define SQLITE_Fts3Tokenizer 0x00400000 /* Enable fts3_tokenizer(2) */ #define SQLITE_EnableQPSG 0x00800000 /* Query Planner Stability Guarantee */ +#define SQLITE_NoopUpdate 0x01000000 /* UPDATE operations are no-ops */ /* Flags used only if debugging */ #ifdef SQLITE_DEBUG -#define SQLITE_NoopUpdate 0x04000000 /* UPDATE operations are no-ops */ #define SQLITE_SqlTrace 0x08000000 /* Debug print SQL as it executes */ #define SQLITE_VdbeListing 0x10000000 /* Debug listings of VDBE programs */ #define SQLITE_VdbeTrace 0x20000000 /* True to trace VDBE execution */ From c0f69fcec77830092331cc4bbc0b6b36d17a5e7a Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 6 Nov 2017 10:04:45 +0000 Subject: [PATCH 083/179] Allow "BEGIN CONCURRENT" transactions to modify the temp schema. FossilOrigin-Name: 0fb6d91cea347384fc081ce4c79582b365801dd4f56f5bf2ed40922bbfca0344 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/vdbe.c | 4 ++-- test/concurrent.test | 36 ++++++++++++++++++++++++++++++++---- 4 files changed, 42 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 861009c901..6930c352bb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Cherrypick\s[ec37ad6d08]\sinto\sthis\sbranch.\sWith\sthis\spatch,\sif\nSQLITE_SHARED_MAPPING\sis\sdefined\sat\sbuild-time\sSQLite\swill\suse\sa\ssingle\smemory\nmapping\sfor\smultiple\sconnections\sto\sthe\ssame\sdatabase\sfile\swithin\sa\ssingle\nprocess. -D 2017-09-22T11:09:09.656 +C Allow\s"BEGIN\sCONCURRENT"\stransactions\sto\smodify\sthe\stemp\sschema. +D 2017-11-06T10:04:45.041 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f @@ -530,7 +530,7 @@ F src/update.c 5404be9e840717323a69209190cdbc9d0d34adaedaaf1d1a1069babf2c4171c0 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 F src/util.c 5168013cfd937a695d23cce8c67cb07a3dda242d4cb812530ba1148b88e0f159 F src/vacuum.c 07eec96f821c7dcbdca2fadffc6e38ea2c24bf409fcb15fe9fb3ac444d632dfe -F src/vdbe.c 96467c73f53c3cb2b3ad7a73c69d59fb43966d772a7fc70114dde14cf3876dd0 +F src/vdbe.c 8b7313c7012c627a0f6579b0c1d3a42f48ffd187e4beb6a0face21092d728ac0 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1 @@ -668,7 +668,7 @@ F test/collateB.test 1e68906951b846570f29f20102ed91d29e634854ee47454d725f2151eca F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 F test/colname.test c47639d26cbeba6977457e5ef2c2c55c5b6c889478dd7eb0ed858ba894e7fa93 F test/concfault.test 500f17c3fcfe7705114422bcc6ddd3c740001a43 -F test/concurrent.test a801cd60c370f0ed851657c9576b102f9ab1dd846c6a88e6ae45939a8deeda7c +F test/concurrent.test 86661967a680670127a62a819e60dc93c2d3d49043ac95b26dfa70d3e60dbde5 F test/concurrent2.test 9dfbeb0a323733fe1d13443371734bb94a674dbf777f464365475903873111f8 F test/concurrent3.test f4af1cf1220908c6dd5694923621c19e999b78cd997e2646285f08a52bcb4170 F test/concurrent4.test e0b12cd467137e50259df3b4f837507e82aaa07c35941c88664dc8ed1d089c44 @@ -1666,8 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 307b802e8627c93a51e4c54851a4fab33db5061bb80e3d327ce53b127d6d511b -Q +ec37ad6d08362f4c9faad9b629c0fa23f5864ff6ad7f4cbed93a25d5f7b815d8 -R a7069c27bdffb405c8574ad991612563 +P c7a5880d6d898299b4c9414b7702cfa450aa5f7bf4ec8f417b94d2a7b6558264 +R 202ef8632fda2b1b9ef7af4f1e374f95 U dan -Z 9a1235c2d010c6dd3774c97b07deb6f2 +Z d758dc3efdc3027768a35f11fc7171bf diff --git a/manifest.uuid b/manifest.uuid index da8f0bb88e..e36be45a6d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c7a5880d6d898299b4c9414b7702cfa450aa5f7bf4ec8f417b94d2a7b6558264 \ No newline at end of file +0fb6d91cea347384fc081ce4c79582b365801dd4f56f5bf2ed40922bbfca0344 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 09c0cd32b4..4292fce260 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3306,7 +3306,7 @@ case OP_SetCookie: { rc = sqlite3BtreeUpdateMeta(pDb->pBt, pOp->p2, pOp->p3); if( pOp->p2==BTREE_SCHEMA_VERSION ){ /* When the schema cookie changes, record the new cookie internally */ - assert( db->bConcurrent==0 ); + assert( pOp->p1==1 || db->bConcurrent==0 ); pDb->pSchema->schema_cookie = pOp->p3; db->mDbFlags |= DBFLAG_SchemaChange; }else if( pOp->p2==BTREE_FILE_FORMAT ){ @@ -6509,7 +6509,7 @@ case OP_Expire: { case OP_TableLock: { u8 isWriteLock = (u8)pOp->p3; #ifndef SQLITE_OMIT_CONCURRENT - if( isWriteLock && db->bConcurrent && pOp->p2==1 ){ + if( isWriteLock && db->bConcurrent && pOp->p2==1 && pOp->p1!=1 ){ rc = SQLITE_ERROR; sqlite3VdbeError(p, "cannot modify database schema within CONCURRENT transaction"); diff --git a/test/concurrent.test b/test/concurrent.test index b8f8d7e0d6..b94071a8f1 100644 --- a/test/concurrent.test +++ b/test/concurrent.test @@ -112,19 +112,47 @@ foreach {tn sql} { 2 { DROP TABLE t1 } 3 { CREATE INDEX i1 ON t1(a) } 4 { CREATE VIEW v1 AS SELECT * FROM t1 } - 5 { CREATE TEMP TABLE xx(a, b) } } { - do_catchsql_test 1.7.$tn.1 " + do_catchsql_test 1.7.0.$tn.1 " BEGIN CONCURRENT; $sql " {1 {cannot modify database schema within CONCURRENT transaction}} - do_execsql_test 1.7.$tn.2 { + do_execsql_test 1.7.0.$tn.2 { SELECT sql FROM sqlite_master; SELECT sql FROM sqlite_temp_master; } {{CREATE TABLE t1(a, b)}} - do_execsql_test 1.7.$tn.3 COMMIT + do_execsql_test 1.7.0.$tn.3 COMMIT +} + +# Except the temp db schema. +foreach {tn sql} { + 1 { CREATE TEMP TABLE xx(a, b) } + 2 { DROP TABLE xx } + 3 { CREATE TEMP TABLE yy(a, b) } + 4 { CREATE VIEW temp.v1 AS SELECT * FROM t1 } + 5 { CREATE INDEX yyi1 ON yy(a); } + 6 { CREATE TABLE temp.zz(a, b) } +} { + do_catchsql_test 1.7.1.$tn.1 " + BEGIN CONCURRENT; + $sql + " {0 {}} + + do_execsql_test 1.7.1.$tn.2 COMMIT +} + + +do_execsql_test 1.7.1.x { + SELECT sql FROM sqlite_master; + SELECT sql FROM sqlite_temp_master; +} { + {CREATE TABLE t1(a, b)} + {CREATE TABLE yy(a, b)} + {CREATE VIEW v1 AS SELECT * FROM t1} + {CREATE INDEX yyi1 ON yy(a)} + {CREATE TABLE zz(a, b)} } #------------------------------------------------------------------------- From cb0267185c8178ad94b9dc169a2ee043ec60321e Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 11 Dec 2017 14:02:10 +0000 Subject: [PATCH 084/179] Abort on an invalid paramater to sqlite3BitvecSet(). FossilOrigin-Name: 163c870950f386f6b0bb1ff9b3886cf95ba0deed414cae75baf87621ed3528c2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/bitvec.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 77220b7e11..dce0a2ddfa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\schanges\sfrom\strunk.\sThis\sfixes\sthe\sSQLITE_ENABLE_UPDATE_DELETE_LIMIT\sfunctionality\sso\sthat\sit\sworks\swith\sviews\sand\sWITHOUT\sROWID\stables. -D 2017-11-14T20:06:15.563 +C Abort\son\san\sinvalid\sparamater\sto\ssqlite3BitvecSet(). +D 2017-12-11T14:02:10.998 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -414,7 +414,7 @@ F src/analyze.c 0d0ccf7520a201d8747ea2f02c92c26e26f801bc161f714f27b9f7630dde0421 F src/attach.c 07b706e336fd3cedbd855e1f8266d10e82fecae07daf86717b5760cd7784c584 F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b -F src/bitvec.c c77b7f5759e413c1c8b53267d633c952e66db79c1171964c7e24c0f92f5019cf +F src/bitvec.c 8433d9e98dd6f2ea3286e0d2fe5d65de1bfc18a706486eb2026b01be066b5806 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca F src/btree.c a41d580524a4cd9b00443b7196d298e05463f6bfcb712853db3abcddc93cf3ab F src/btree.h feafd0647331366f4ef17f7e68597e9029f001e7ab16a125e2f176c598a7ef4a @@ -1686,7 +1686,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7f217edab4575554f657d38e2a1bc6b3f577998fdbecb04eb200aeb8b8406581 dae4a97a483bee1e6ac0271ddd28a0dffcebf7522edaf12eb5e0eba5fc62516a -R d370973eeca86ae526a0060a468b38d6 -U dan -Z 20f27c4758e319217032ba89b2abf6bd +P d90e5f346bcf7adab26ca8dad9dfbd0fbb86604a15f2fe827f11b3faab036750 +R e43dfcde6e63fd81287681f03e602a2f +U drh +Z 3fc04e05119313efe5c6f01b2d450caa diff --git a/manifest.uuid b/manifest.uuid index b87d30350a..af787c32f4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d90e5f346bcf7adab26ca8dad9dfbd0fbb86604a15f2fe827f11b3faab036750 \ No newline at end of file +163c870950f386f6b0bb1ff9b3886cf95ba0deed414cae75baf87621ed3528c2 \ No newline at end of file diff --git a/src/bitvec.c b/src/bitvec.c index 31994f550b..68ff12dc91 100644 --- a/src/bitvec.c +++ b/src/bitvec.c @@ -175,6 +175,7 @@ int sqlite3BitvecSet(Bitvec *p, u32 i){ sqlite3_log(SQLITE_ERROR, "Bitvec: setting bit %d of bitvec size %d\n", (int)i, (int)p->iSize ); + abort(); } i--; while((p->iSize > BITVEC_NBIT) && p->iDivisor) { From 55fba4f05cd32ca2e4a9077e62ce5987537f3823 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 2 Jan 2018 19:57:26 +0000 Subject: [PATCH 085/179] Fix a spurious SQLITE_CORRUPT error that could occur within a COMMIT of a concurrent transaction. FossilOrigin-Name: 50c8952c92b9f0c61935fb0df04ed1426d9e266a812071b7bf5b0215c5552757 --- manifest | 13 +++++----- manifest.uuid | 2 +- src/btree.c | 6 +++-- test/concurrent6.test | 60 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 72 insertions(+), 9 deletions(-) create mode 100644 test/concurrent6.test diff --git a/manifest b/manifest index c25d494650..77dbf33fb4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\sbegin-concurrent\schanges\sinto\sthis\sbranch. -D 2017-12-12T18:17:09.710 +C Fix\sa\sspurious\sSQLITE_CORRUPT\serror\sthat\scould\soccur\swithin\sa\sCOMMIT\sof\sa\nconcurrent\stransaction. +D 2018-01-02T19:57:26.765 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -416,7 +416,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 8433d9e98dd6f2ea3286e0d2fe5d65de1bfc18a706486eb2026b01be066b5806 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c a41d580524a4cd9b00443b7196d298e05463f6bfcb712853db3abcddc93cf3ab +F src/btree.c c3e15a9e5726fa3f426322cccaf9fe972dc9b3d9e07921220286f9076413f71d F src/btree.h feafd0647331366f4ef17f7e68597e9029f001e7ab16a125e2f176c598a7ef4a F src/btreeInt.h 0e0abe97427b4139092ec8782d396a4ad18566964e992c60043e370d4c86fd99 F src/build.c f890a66f2b78cd820b21b580f37605f8dd77f19d0b35f5850a675c88a815adca @@ -686,6 +686,7 @@ F test/concurrent2.test 9dfbeb0a323733fe1d13443371734bb94a674dbf777f464365475903 F test/concurrent3.test f4af1cf1220908c6dd5694923621c19e999b78cd997e2646285f08a52bcb4170 F test/concurrent4.test e0b12cd467137e50259df3b4f837507e82aaa07c35941c88664dc8ed1d089c44 F test/concurrent5.test d5d7d9d404a9b4502464fc097c1fc5c3012bb4f1b063fae7ad707ca983fc86c5 +F test/concurrent6.test a7860e9ca13bb5fb76bcf41c5524fbfa9c37e6e258ecf84ffb5748a272488c67 F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c F test/conflict3.test a83db76a6c3503b2fa057c7bfb08c318d8a422202d8bc5b86226e078e5b49ff9 @@ -1686,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bdf791f9f7f7ab4b1871e1d3a0d5edcad46fc67d7336ae7f3f7b20e69801be8e 163c870950f386f6b0bb1ff9b3886cf95ba0deed414cae75baf87621ed3528c2 -R 99497e88454df0f5b4640ad740c855cd +P 3fde0b4d05c249b9d2a54dd721185202c353cee23c0351b634cac349dc0a7b14 +R 131f3c49cbacffd5469d66a498022e8f U dan -Z 55bd54709eec1e2f156a3a0d8460b930 +Z 25c8423f0f81f25c528eab1972eef804 diff --git a/manifest.uuid b/manifest.uuid index dfe596be48..bb2728f213 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3fde0b4d05c249b9d2a54dd721185202c353cee23c0351b634cac349dc0a7b14 \ No newline at end of file +50c8952c92b9f0c61935fb0df04ed1426d9e266a812071b7bf5b0215c5552757 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index e46a3f2c34..e3fd5a3928 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4179,10 +4179,12 @@ static int btreeFixUnlocked(Btree *p){ /* The current transaction allocated pages pMap->iFirst through ** nPage (inclusive) at the end of the database file. Meanwhile, ** other transactions have allocated (iFirst..nHPage). So move - ** pages (iFirst..MIN(nPage,nHPage)) to (MAX(nPage,nHPage)+1). */ + ** pages (iFirst..MIN(nPage,nHPage)) to (MAX(nPage,nHPage)+1). */ Pgno iLast = MIN(nPage, nHPage); /* Last page to move */ Pgno nCurrent; /* Current size of db */ + nCurrent = MAX(nPage, nHPage); + pBt->nPage = nCurrent; rc = btreeRelocateRange(pBt, pMap->iFirst, iLast, &nCurrent); /* There are now no collisions with the snapshot at the head of the @@ -6127,7 +6129,7 @@ static int allocateBtreePage( ** stores stores the total number of pages on the freelist. */ n = get4byte(&pPage1->aData[36]); testcase( n==mxPage-1 ); - if( ISCONCURRENT==0 && n>=mxPage ){ + if( n>=mxPage ){ return SQLITE_CORRUPT_BKPT; } diff --git a/test/concurrent6.test b/test/concurrent6.test new file mode 100644 index 0000000000..44718b7dbc --- /dev/null +++ b/test/concurrent6.test @@ -0,0 +1,60 @@ +# 2017 May 26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/wal_common.tcl +set ::testprefix concurrent6 + +ifcapable !concurrent { + finish_test + return +} + +sqlite3 db2 test.db + +do_execsql_test 1.0 { + PRAGMA page_size = 1024; + PRAGMA journal_mode = wal; + CREATE TABLE t1(x); + CREATE TABLE t2(x); + CREATE TABLE t3(x); + CREATE TABLE t4(x); + + INSERT INTO t1 VALUES(zeroblob(1500)); +} {wal} + +do_execsql_test -db db2 1.1 { + BEGIN CONCURRENT; + INSERT INTO t3 VALUES(zeroblob(4000)); + DELETE FROM t1; +} + +do_execsql_test 1.2 { + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100) + INSERT INTO t2 SELECT zeroblob(1000) FROM s; + + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100) + INSERT INTO t4 SELECT zeroblob(1000) FROM s; + + DELETE FROM t4; +} + +do_execsql_test -db db2 1.3 { + COMMIT; +} + + +finish_test + From 5d9c916150203985051a35a3e9708c5b68d51a37 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 4 Jan 2018 18:36:39 +0000 Subject: [PATCH 086/179] Fix problem causing free-list corruption when merging free-lists for two concurrent transactions that have both used page X as an in-memory free-list trunk page, where X lies past the end of the initial database images. FossilOrigin-Name: dc0fc2aa7cbefeb5f0ba8c992fd3e9adcfb5a4d61e2321c1bd93f4d36ba9aafc --- manifest | 13 ++++++----- manifest.uuid | 2 +- src/btree.c | 5 ++++- test/concurrent7.test | 52 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 8 deletions(-) create mode 100644 test/concurrent7.test diff --git a/manifest b/manifest index 77dbf33fb4..8f46fe4a5f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sspurious\sSQLITE_CORRUPT\serror\sthat\scould\soccur\swithin\sa\sCOMMIT\sof\sa\nconcurrent\stransaction. -D 2018-01-02T19:57:26.765 +C Fix\sproblem\scausing\sfree-list\scorruption\swhen\smerging\sfree-lists\sfor\stwo\nconcurrent\stransactions\sthat\shave\sboth\sused\spage\sX\sas\san\sin-memory\sfree-list\ntrunk\spage,\swhere\sX\slies\spast\sthe\send\sof\sthe\sinitial\sdatabase\simages. +D 2018-01-04T18:36:39.212 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -416,7 +416,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 8433d9e98dd6f2ea3286e0d2fe5d65de1bfc18a706486eb2026b01be066b5806 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c c3e15a9e5726fa3f426322cccaf9fe972dc9b3d9e07921220286f9076413f71d +F src/btree.c 8cbe0ce25607b0a64b44dbf9d018d0d667569b5966ad1bd40f27f95d65ce4284 F src/btree.h feafd0647331366f4ef17f7e68597e9029f001e7ab16a125e2f176c598a7ef4a F src/btreeInt.h 0e0abe97427b4139092ec8782d396a4ad18566964e992c60043e370d4c86fd99 F src/build.c f890a66f2b78cd820b21b580f37605f8dd77f19d0b35f5850a675c88a815adca @@ -687,6 +687,7 @@ F test/concurrent3.test f4af1cf1220908c6dd5694923621c19e999b78cd997e2646285f08a5 F test/concurrent4.test e0b12cd467137e50259df3b4f837507e82aaa07c35941c88664dc8ed1d089c44 F test/concurrent5.test d5d7d9d404a9b4502464fc097c1fc5c3012bb4f1b063fae7ad707ca983fc86c5 F test/concurrent6.test a7860e9ca13bb5fb76bcf41c5524fbfa9c37e6e258ecf84ffb5748a272488c67 +F test/concurrent7.test b96fa5c4cfdf8d5c0bc66b6934214500bad0260884a736f054ccc76e81aae85d F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c F test/conflict3.test a83db76a6c3503b2fa057c7bfb08c318d8a422202d8bc5b86226e078e5b49ff9 @@ -1687,7 +1688,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3fde0b4d05c249b9d2a54dd721185202c353cee23c0351b634cac349dc0a7b14 -R 131f3c49cbacffd5469d66a498022e8f +P 50c8952c92b9f0c61935fb0df04ed1426d9e266a812071b7bf5b0215c5552757 +R 27d77b980c6b081a9288d237e34b2379 U dan -Z 25c8423f0f81f25c528eab1972eef804 +Z 38a96e74ef2f420253bcb0ab74b036ef diff --git a/manifest.uuid b/manifest.uuid index bb2728f213..01f2fea017 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -50c8952c92b9f0c61935fb0df04ed1426d9e266a812071b7bf5b0215c5552757 \ No newline at end of file +dc0fc2aa7cbefeb5f0ba8c992fd3e9adcfb5a4d61e2321c1bd93f4d36ba9aafc \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index e3fd5a3928..9e02e32724 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4095,7 +4095,10 @@ static int btreeRelocateRange( if( pEntry->eType==PTRMAP_FREEPAGE ){ Pgno dummy; rc = allocateBtreePage(pBt, &pFree, &dummy, iPg, BTALLOC_EXACT); - releasePage(pFree); + if( pFree ){ + assert( sqlite3PagerPageRefcount(pFree->pDbPage)==1 ); + sqlite3PcacheDrop(pFree->pDbPage); + } assert( rc!=SQLITE_OK || dummy==iPg ); }else if( pnCurrent ){ btreeGetPage(pBt, iPg, &pPg, 0); diff --git a/test/concurrent7.test b/test/concurrent7.test new file mode 100644 index 0000000000..871e428031 --- /dev/null +++ b/test/concurrent7.test @@ -0,0 +1,52 @@ +# 2018 Jan 5 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix concurrent7 + +sqlite3 db2 test.db + +do_execsql_test 1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x); + CREATE TABLE t2(x); +} {wal} + +do_execsql_test -db db2 2 { + SELECT * FROM t1; +} + +do_execsql_test 3 { + BEGIN CONCURRENT; + INSERT INTO t1 VALUES(randomblob(1500)); + INSERT INTO t1 VALUES(randomblob(1500)); + DELETE FROM t1 WHERE rowid = 1; +} + +do_execsql_test -db db2 4 { + INSERT INTO t2 VALUES(randomblob(1500)); + INSERT INTO t2 VALUES(randomblob(1500)); + INSERT INTO t2 VALUES(randomblob(1500)); + INSERT INTO t2 VALUES(randomblob(1500)); + DELETE FROM t2 WHERE rowid IN (1, 2); +} + +do_execsql_test 5 { + COMMIT; + PRAGMA integrity_check; +} {ok} + +finish_test + + From c7a82772dcd06cc10036e1a84ef6cd2ad9f15dd6 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 20 Feb 2018 21:00:45 +0000 Subject: [PATCH 087/179] Add extra code to log details when corruption is detected in the pointer-map structure maintained by the b-tree layer in begin-concurrent transactions. FossilOrigin-Name: 570233716032f258b878d52c4d5a47e07292d66fa84e3a85c0388ec15efee625 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 8f46fe4a5f..a356e9989c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sproblem\scausing\sfree-list\scorruption\swhen\smerging\sfree-lists\sfor\stwo\nconcurrent\stransactions\sthat\shave\sboth\sused\spage\sX\sas\san\sin-memory\sfree-list\ntrunk\spage,\swhere\sX\slies\spast\sthe\send\sof\sthe\sinitial\sdatabase\simages. -D 2018-01-04T18:36:39.212 +C Add\sextra\scode\sto\slog\sdetails\swhen\scorruption\sis\sdetected\sin\sthe\spointer-map\nstructure\smaintained\sby\sthe\sb-tree\slayer\sin\sbegin-concurrent\stransactions. +D 2018-02-20T21:00:45.475 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -416,7 +416,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 8433d9e98dd6f2ea3286e0d2fe5d65de1bfc18a706486eb2026b01be066b5806 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 8cbe0ce25607b0a64b44dbf9d018d0d667569b5966ad1bd40f27f95d65ce4284 +F src/btree.c 8c6b975926e62c88dfc146c71eb8629e25318a326948b8694de8b87261d5a961 F src/btree.h feafd0647331366f4ef17f7e68597e9029f001e7ab16a125e2f176c598a7ef4a F src/btreeInt.h 0e0abe97427b4139092ec8782d396a4ad18566964e992c60043e370d4c86fd99 F src/build.c f890a66f2b78cd820b21b580f37605f8dd77f19d0b35f5850a675c88a815adca @@ -1688,7 +1688,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 50c8952c92b9f0c61935fb0df04ed1426d9e266a812071b7bf5b0215c5552757 -R 27d77b980c6b081a9288d237e34b2379 +P dc0fc2aa7cbefeb5f0ba8c992fd3e9adcfb5a4d61e2321c1bd93f4d36ba9aafc +R 374752e5ff248b7c36b763a5149c6f74 U dan -Z 38a96e74ef2f420253bcb0ab74b036ef +Z f6098e8b2b3786dd4ba01a06d1e8ac97 diff --git a/manifest.uuid b/manifest.uuid index 01f2fea017..8c7c80619e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dc0fc2aa7cbefeb5f0ba8c992fd3e9adcfb5a4d61e2321c1bd93f4d36ba9aafc \ No newline at end of file +570233716032f258b878d52c4d5a47e07292d66fa84e3a85c0388ec15efee625 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 9e02e32724..2b38656e1f 100644 --- a/src/btree.c +++ b/src/btree.c @@ -628,11 +628,48 @@ static void btreePtrmapDelete(BtShared *pBt){ pBt->pMap = 0; } } + +/* +** Check that the pointer-map does not contain any entries with a parent +** page of 0. Call sqlite3_log() multiple times to output the entire +** data structure if it does. +*/ +static void btreePtrmapCheck(BtShared *pBt, Pgno nPage){ + Pgno i; + int bProblem = 0; + BtreePtrmap *p = pBt->pMap; + + for(i=p->iFirst; i<=nPage; i++){ + PtrmapEntry *pEntry = &p->aPtr[i-p->iFirst]; + if( pEntry->eType==PTRMAP_OVERFLOW1 + || pEntry->eType==PTRMAP_OVERFLOW2 + || pEntry->eType==PTRMAP_BTREE + ){ + if( pEntry->parent==0 ){ + bProblem = 1; + break; + } + } + } + + if( bProblem ){ + for(i=p->iFirst; i<=nPage; i++){ + PtrmapEntry *pEntry = &p->aPtr[i-p->iFirst]; + sqlite3_log(SQLITE_CORRUPT, + "btreePtrmapCheck: pgno=%d eType=%d parent=%d", + (int)i, (int)pEntry->eType, (int)pEntry->parent + ); + } + abort(); + } +} + #else /* SQLITE_OMIT_CONCURRENT */ # define btreePtrmapAllocate(x) SQLITE_OK # define btreePtrmapDelete(x) # define btreePtrmapBegin(x,y) SQLITE_OK # define btreePtrmapEnd(x,y,z) +# define btreePtrmapCheck(y,z) #endif /* SQLITE_OMIT_CONCURRENT */ static void releasePage(MemPage *pPage); /* Forward reference */ @@ -4156,6 +4193,8 @@ static int btreeFixUnlocked(Btree *p){ Pgno iHTrunk = get4byte(&p1[32]); u32 nHFree = get4byte(&p1[36]); + btreePtrmapCheck(pBt, nPage); + /* Attach the head database free list to the end of the current ** transactions free-list (if any). */ if( iTrunk!=0 ){ From cdbf6d01a49b2658e028ebcbb27a631d37243a11 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 20 Mar 2018 13:54:22 +0000 Subject: [PATCH 088/179] Remove debugging puts from concurrrent3.test. FossilOrigin-Name: 13b4975681f249831a22562d1c26958f841ea77ca779858b0bee735dd30b710e --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/concurrent3.test | 12 +++++++----- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 25807c5b4c..fc7088ced9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\srecent\senhancements\sfrom\strunk. -D 2018-03-20T13:52:42.504 +C Remove\sdebugging\sputs\sfrom\sconcurrrent3.test. +D 2018-03-20T13:54:22.689 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7016fc56c6b9bfe5daac4f34be8be38d8c0b5fab79ccbfb764d3b23bf1c6fff3 @@ -702,7 +702,7 @@ F test/colname.test fb28b3687e03625425bc216edf8b186ce974aa71008e2aa1f426a7dcb75a F test/concfault.test 500f17c3fcfe7705114422bcc6ddd3c740001a43 F test/concurrent.test 86661967a680670127a62a819e60dc93c2d3d49043ac95b26dfa70d3e60dbde5 F test/concurrent2.test 9dfbeb0a323733fe1d13443371734bb94a674dbf777f464365475903873111f8 -F test/concurrent3.test f4af1cf1220908c6dd5694923621c19e999b78cd997e2646285f08a52bcb4170 +F test/concurrent3.test 530671ac706f6a1d0f4992dbdd33a86408330d03cd90fb9e82ecb1b27f5fd081 F test/concurrent4.test e0b12cd467137e50259df3b4f837507e82aaa07c35941c88664dc8ed1d089c44 F test/concurrent5.test d5d7d9d404a9b4502464fc097c1fc5c3012bb4f1b063fae7ad707ca983fc86c5 F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db @@ -1723,7 +1723,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fb6b7938601505186c0b1f0df6c45630039027f85ff394d290bc2c86b16a3a07 ec7addc87f97bcff3c3694b14a680453b52de3f8c106436f0708a1cc04b90faa -R 43c5e6204e5e1e748f1b8270da37c665 +P b0c2f760a637ee973f4dcc27308eec44950e6d0a9c5ab5c828c1210c1f868efa +R ca9cb1d929805c71db9b6a05a48b9513 U drh -Z dfd53febd46dc841536127bb8e42a571 +Z ae9c4938206a0b6e0c232c0b2ca9588b diff --git a/manifest.uuid b/manifest.uuid index e069a3d23c..4412f52407 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b0c2f760a637ee973f4dcc27308eec44950e6d0a9c5ab5c828c1210c1f868efa \ No newline at end of file +13b4975681f249831a22562d1c26958f841ea77ca779858b0bee735dd30b710e \ No newline at end of file diff --git a/test/concurrent3.test b/test/concurrent3.test index 4364f84012..cc80388902 100644 --- a/test/concurrent3.test +++ b/test/concurrent3.test @@ -27,7 +27,10 @@ ifcapable !concurrent { db close sqlite3_shutdown test_sqlite3_log xLog -proc xLog {error_code msg} { puts "$error_code: $msg" } +proc xLog {error_code msg} { + # puts "$error_code: $msg" + # Enable the previous for debugging +} reset_db proc create_schema {} { @@ -218,12 +221,11 @@ foreach {tn nRepeat oplist} { foreach db $DBLIST { $db close } - foreach k [lsort [array names used]] { - puts "$k: $stats($k,0) committed, $stats($k,1) rolled back" - } + # foreach k [lsort [array names used]] { + # puts "$k: $stats($k,0) committed, $stats($k,1) rolled back" + # } } finish_test - From bfbe2b8c80ac95f12d824dd0bb5cc64d2f555c77 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 15 May 2018 08:51:05 +0000 Subject: [PATCH 089/179] Include the value of the "flags" byte of the relevant page in the log message emitted when a BEGIN CONCURRENT commit conflict is detected. FossilOrigin-Name: fbfa547177ac7d57dcb0775d761a7f92fc1f2263cd83e2843cd109139a8c5121 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 7 +++++-- test/concurrent5.test | 8 ++++---- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 78138b0dbf..b989d2b3f2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\schanges\sfrom\strunk.\sIncluding\sthe\s"ORDER\sBY\s...\sLIMIT"\noptimization. -D 2018-04-26T17:54:11.487 +C Include\sthe\svalue\sof\sthe\s"flags"\sbyte\sof\sthe\srelevant\spage\sin\sthe\slog\smessage\nemitted\swhen\sa\sBEGIN\sCONCURRENT\scommit\sconflict\sis\sdetected. +D 2018-05-15T08:51:05.745 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 5ce9343cba9c189046f1afe6d2bcc1f68079439febc05267b98aec6ecc752439 @@ -437,7 +437,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 8433d9e98dd6f2ea3286e0d2fe5d65de1bfc18a706486eb2026b01be066b5806 F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 -F src/btree.c a64d302f0fa956af74d2a74538d206c3c854bdb9c6d53b92825b774e8465bcb6 +F src/btree.c 593a90431223ca1dcdd3ff6797e96d36147434431dbfaf213e866e400dd408df F src/btree.h 0cd745755efd1f3df4c70544c54253920ea32fe6b179b97e9daeb786ba0de4ba F src/btreeInt.h 6c65e6c96f561596f6870c79a64d4706af81613881d7947e3f063e923f14115f F src/build.c 1ef8945748702f0069521bafe33df41647bea777fd2353c73530d522a9fedaf8 @@ -708,7 +708,7 @@ F test/concurrent.test 86661967a680670127a62a819e60dc93c2d3d49043ac95b26dfa70d3e F test/concurrent2.test 9dfbeb0a323733fe1d13443371734bb94a674dbf777f464365475903873111f8 F test/concurrent3.test 530671ac706f6a1d0f4992dbdd33a86408330d03cd90fb9e82ecb1b27f5fd081 F test/concurrent4.test e0b12cd467137e50259df3b4f837507e82aaa07c35941c88664dc8ed1d089c44 -F test/concurrent5.test d5d7d9d404a9b4502464fc097c1fc5c3012bb4f1b063fae7ad707ca983fc86c5 +F test/concurrent5.test ff73e642342f02133151c23b0eecab2ec1ea5bc9645c646861a9baa4263d8f50 F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c F test/conflict3.test a83db76a6c3503b2fa057c7bfb08c318d8a422202d8bc5b86226e078e5b49ff9 @@ -1736,7 +1736,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b27bd799ea66748b89223044c536813d56d0aa513f077499cbf8ed9c974d7b01 0fcfc36ceb820fc70136b799a0405fe92e50646e697be2872bbe9a53a05ed5a9 -R aacb8135e5d5d61c420d0eb994490898 +P d8ae7ba083551f05b963f13c5164ff60257c7b615cd48d675027cb6289b463da +R b31f3045e7e18d43f49b3d4aef3b5822 U dan -Z 0c0899207e7f0c3c73b0ec6107e8e193 +Z bf3fce17ea47da69e27674ff4f78c20a diff --git a/manifest.uuid b/manifest.uuid index 9dcfb8e644..285075f650 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d8ae7ba083551f05b963f13c5164ff60257c7b615cd48d675027cb6289b463da \ No newline at end of file +fbfa547177ac7d57dcb0775d761a7f92fc1f2263cd83e2843cd109139a8c5121 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 013f4cb6f0..2f4d82e5a6 100644 --- a/src/btree.c +++ b/src/btree.c @@ -10433,6 +10433,7 @@ int sqlite3BtreeExclusiveLock(Btree *p){ PgHdr *pPg = 0; int rc2 = sqlite3PagerGet(pBt->pPager, pgno, &pPg, 0); if( rc2==SQLITE_OK ){ + u8 pageFlags = 0; int bWrite = -1; const char *zObj = 0; const char *zTab = 0; @@ -10444,6 +10445,7 @@ int sqlite3BtreeExclusiveLock(Btree *p){ pgnoRoot = ((MemPage*)sqlite3PagerGetExtra(pPg))->pgnoRoot; bWrite = sqlite3PagerIswriteable(pPg); + pageFlags = ((u8*)sqlite3PagerGetData(pPg))[0]; sqlite3PagerUnref(pPg); pSchema = sqlite3SchemaGet(p->db, p); @@ -10469,11 +10471,12 @@ int sqlite3BtreeExclusiveLock(Btree *p){ sqlite3_log(SQLITE_OK, "cannot commit CONCURRENT transaction " "- conflict at page %d " - "(%s page; part of db %s %s%s%s)", + "(%s page; part of db %s %s%s%s; flags=0x%02x)", (int)pgno, (bWrite==0?"read-only":(bWrite>0?"read/write":"unknown")), (zTab ? "index" : "table"), - (zTab ? zTab : ""), (zTab ? "." : ""), (zObj ? zObj : "UNKNOWN") + (zTab ? zTab : ""), (zTab ? "." : ""), (zObj ? zObj : "UNKNOWN"), + pageFlags ); } } diff --git a/test/concurrent5.test b/test/concurrent5.test index dde661ea8b..1747e2ed44 100644 --- a/test/concurrent5.test +++ b/test/concurrent5.test @@ -59,7 +59,7 @@ do_test 1.1.1 { catchsql COMMIT db2 } {1 {database is locked}} do_test_conflict_msg 1.1.2 { - conflict at page 2 (read-only page; part of db table t1) + conflict at page 2 (read-only page; part of db table t1; flags=0x05) } do_test 1.2.1 { @@ -78,7 +78,7 @@ do_test 1.2.1 { } {1 {database is locked}} do_test_conflict_msg 1.2.2 { - conflict at page 105 (read/write page; part of db table t1) + conflict at page 105 (read/write page; part of db table t1; flags=0x0d) } do_test 1.3.1 { @@ -97,7 +97,7 @@ do_test 1.3.1 { } {1 {database is locked}} do_test_conflict_msg 1.3.2 { - conflict at page 3 (read/write page; part of db table t2) + conflict at page 3 (read/write page; part of db table t2; flags=0x0d) } do_test 1.4.1 { @@ -121,7 +121,7 @@ do_test 1.4.1 { } {1 {database is locked}} do_test_conflict_msg 1.3.2 { - conflict at page 211 (read/write page; part of db index t3.i3) + conflict at page 211 (read/write page; part of db index t3.i3; flags=0x0a) } db close From b7ee5667c29f08436370b2043a74e2608392f534 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 15 May 2018 11:28:36 +0000 Subject: [PATCH 090/179] Instead of just the flags byte, include the first 8 bytes of the relevant page in an on-commit conflict log message. FossilOrigin-Name: e7dc03e7439f224e4bec458f27fe364f2fb1ac637328657a578ada693ae22842 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 18 ++++++++++++++---- test/concurrent5.test | 8 ++++---- 4 files changed, 26 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index b989d2b3f2..3cd10ab329 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Include\sthe\svalue\sof\sthe\s"flags"\sbyte\sof\sthe\srelevant\spage\sin\sthe\slog\smessage\nemitted\swhen\sa\sBEGIN\sCONCURRENT\scommit\sconflict\sis\sdetected. -D 2018-05-15T08:51:05.745 +C Instead\sof\sjust\sthe\sflags\sbyte,\sinclude\sthe\sfirst\s8\sbytes\sof\sthe\srelevant\spage\nin\san\son-commit\sconflict\slog\smessage. +D 2018-05-15T11:28:36.683 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 5ce9343cba9c189046f1afe6d2bcc1f68079439febc05267b98aec6ecc752439 @@ -437,7 +437,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 8433d9e98dd6f2ea3286e0d2fe5d65de1bfc18a706486eb2026b01be066b5806 F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 -F src/btree.c 593a90431223ca1dcdd3ff6797e96d36147434431dbfaf213e866e400dd408df +F src/btree.c 78be543c894d6fb6e1be538251246ae6da35d63ab29bd65dbf27a62d73444358 F src/btree.h 0cd745755efd1f3df4c70544c54253920ea32fe6b179b97e9daeb786ba0de4ba F src/btreeInt.h 6c65e6c96f561596f6870c79a64d4706af81613881d7947e3f063e923f14115f F src/build.c 1ef8945748702f0069521bafe33df41647bea777fd2353c73530d522a9fedaf8 @@ -708,7 +708,7 @@ F test/concurrent.test 86661967a680670127a62a819e60dc93c2d3d49043ac95b26dfa70d3e F test/concurrent2.test 9dfbeb0a323733fe1d13443371734bb94a674dbf777f464365475903873111f8 F test/concurrent3.test 530671ac706f6a1d0f4992dbdd33a86408330d03cd90fb9e82ecb1b27f5fd081 F test/concurrent4.test e0b12cd467137e50259df3b4f837507e82aaa07c35941c88664dc8ed1d089c44 -F test/concurrent5.test ff73e642342f02133151c23b0eecab2ec1ea5bc9645c646861a9baa4263d8f50 +F test/concurrent5.test 0c16cbf7446af162a14e6def30445e94016064eb994e5aa4ebb2bebc59554176 F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c F test/conflict3.test a83db76a6c3503b2fa057c7bfb08c318d8a422202d8bc5b86226e078e5b49ff9 @@ -1736,7 +1736,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d8ae7ba083551f05b963f13c5164ff60257c7b615cd48d675027cb6289b463da -R b31f3045e7e18d43f49b3d4aef3b5822 +P fbfa547177ac7d57dcb0775d761a7f92fc1f2263cd83e2843cd109139a8c5121 +R fdff9a25ded182bc4a0b3f2b7933cc53 U dan -Z bf3fce17ea47da69e27674ff4f78c20a +Z acdbc76c6ad1bafb932ec798f2a5510e diff --git a/manifest.uuid b/manifest.uuid index 285075f650..0fdab1824d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fbfa547177ac7d57dcb0775d761a7f92fc1f2263cd83e2843cd109139a8c5121 \ No newline at end of file +e7dc03e7439f224e4bec458f27fe364f2fb1ac637328657a578ada693ae22842 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 2f4d82e5a6..35602cec08 100644 --- a/src/btree.c +++ b/src/btree.c @@ -10433,19 +10433,29 @@ int sqlite3BtreeExclusiveLock(Btree *p){ PgHdr *pPg = 0; int rc2 = sqlite3PagerGet(pBt->pPager, pgno, &pPg, 0); if( rc2==SQLITE_OK ){ - u8 pageFlags = 0; int bWrite = -1; const char *zObj = 0; const char *zTab = 0; + char zContent[17]; if( pPg ){ Pgno pgnoRoot = 0; HashElem *pE; Schema *pSchema; + u8 *aData = (u8*)sqlite3PagerGetData(pPg); + int i; + for(i=0; i<8; i++){ + static const char hexdigits[] = { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' + }; + zContent[i*2] = hexdigits[(aData[i] >> 4)]; + zContent[i*2+1] = hexdigits[(aData[i] & 0xF)]; + } + zContent[16] = '\0'; pgnoRoot = ((MemPage*)sqlite3PagerGetExtra(pPg))->pgnoRoot; bWrite = sqlite3PagerIswriteable(pPg); - pageFlags = ((u8*)sqlite3PagerGetData(pPg))[0]; sqlite3PagerUnref(pPg); pSchema = sqlite3SchemaGet(p->db, p); @@ -10471,12 +10481,12 @@ int sqlite3BtreeExclusiveLock(Btree *p){ sqlite3_log(SQLITE_OK, "cannot commit CONCURRENT transaction " "- conflict at page %d " - "(%s page; part of db %s %s%s%s; flags=0x%02x)", + "(%s page; part of db %s %s%s%s; content=%s...)", (int)pgno, (bWrite==0?"read-only":(bWrite>0?"read/write":"unknown")), (zTab ? "index" : "table"), (zTab ? zTab : ""), (zTab ? "." : ""), (zObj ? zObj : "UNKNOWN"), - pageFlags + zContent ); } } diff --git a/test/concurrent5.test b/test/concurrent5.test index 1747e2ed44..bc00232442 100644 --- a/test/concurrent5.test +++ b/test/concurrent5.test @@ -59,7 +59,7 @@ do_test 1.1.1 { catchsql COMMIT db2 } {1 {database is locked}} do_test_conflict_msg 1.1.2 { - conflict at page 2 (read-only page; part of db table t1; flags=0x05) + conflict at page 2 (read-only page; part of db table t1; content=0500000063021100...) } do_test 1.2.1 { @@ -78,7 +78,7 @@ do_test 1.2.1 { } {1 {database is locked}} do_test_conflict_msg 1.2.2 { - conflict at page 105 (read/write page; part of db table t1; flags=0x0d) + conflict at page 105 (read/write page; part of db table t1; content=0D00000002026100...) } do_test 1.3.1 { @@ -97,7 +97,7 @@ do_test 1.3.1 { } {1 {database is locked}} do_test_conflict_msg 1.3.2 { - conflict at page 3 (read/write page; part of db table t2; flags=0x0d) + conflict at page 3 (read/write page; part of db table t2; content=0D0000000103FB00...) } do_test 1.4.1 { @@ -121,7 +121,7 @@ do_test 1.4.1 { } {1 {database is locked}} do_test_conflict_msg 1.3.2 { - conflict at page 211 (read/write page; part of db index t3.i3; flags=0x0a) + conflict at page 211 (read/write page; part of db index t3.i3; content=0A0310006300D800...) } db close From 9b5c67f78485dbef431d996577bdc5b9f4371ffb Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 30 Nov 2018 16:26:39 +0000 Subject: [PATCH 091/179] Merge the mutex-free PRNG change into this branch. FossilOrigin-Name: 81e626f4a44be75425cf916ec61b6b36df0907ebac4adbf6786f87ad4f3a0674 --- manifest | 23 ++++++++++++----------- manifest.uuid | 2 +- src/func.c | 6 ++++-- src/main.c | 2 +- src/random.c | 22 ++++++++++++++++++++++ src/select.c | 2 +- src/sqliteInt.h | 12 ++++++++++++ src/wal.c | 8 +++++--- 8 files changed, 58 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index e89604258e..4fafda0823 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\sbegin-concurrent\schanges\sinto\sthis\sbranch. -D 2018-11-26T07:34:33.644 +C Merge\sthe\smutex-free\sPRNG\schange\sinto\sthis\sbranch. +D 2018-11-30T16:26:39.800 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in b730006b54c990461d864c5387f2e6f13aadb0236804555fb010ed6865a5f058 @@ -465,7 +465,7 @@ F src/delete.c cec65c0e74be7492cafba1b77580732b0b1a41a4dbc4ac70909ac44b65b2a20b F src/expr.c 9aacc0b72348ba90010b672dcbbbe2fa56e1182043bc917a3a147b2bc57a5497 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 972a4ba14296bef2303a0abbad1e3d82bc3c61f9e6ce4e8e9528bdee68748812 -F src/func.c 7c288b4ce309b5a8b8473514b88e1f8e69a80134509a8c0db8e39c858e367e7f +F src/func.c 8efa2c813b3f6a831a070311b5bcbc97993b79cbcd274bdb49bde56ccd3d37bc F src/global.c 9bf034fd560bdd514715170ed8460bb7f823cec113f0569ef3f18a20c7ccd128 F src/hash.c 931ec82d7e070654a8facb42549bbb3a25720171d73ba94c3d3160580d01ef1f F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 @@ -474,7 +474,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 6b81aae27b196925d8ff78824f4bbd435d6a40cd38dc324685e21735bb402109 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e F src/loadext.c 448eab53ecdb566a1259ee2d45ebff9c0bc4a2cf393774488775c33e4fbe89bf -F src/main.c 47b8f0411812b3dd7c41fddf09420638aabfb838c6134de0ec5c18df381ecd28 +F src/main.c b23c7b1eeca05ee3af56dacdc0efdf61484dc17194c82233fb98b42ee8e807b8 F src/malloc.c 07295435093ce354c6d9063ac05a2eeae28bd251d2e63c48b3d67c12c76f7e18 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -507,15 +507,15 @@ F src/pragma.c a2eab23fbf7c70c28f3a22629de2662d052c22853649430be302b910719574de F src/pragma.h 7003ea8e45e5da0a7cd6d35846214f9ae9ecf5be66b268415ceea5855324af11 F src/prepare.c f81f8d707e583192c28fea0b2e19385415b7d188123b23f49b038076408d7a69 F src/printf.c 0f1177cf1dd4d7827bf64d840768514ec76409abecaca9e8b577dbd065150381 -F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 +F src/random.c f27af4099afaea7284ade5c206224dcfdb2334cfd119d018b470d46356b3f27d F src/resolve.c bc8c79e56439b111e7d9415e44940951f7087e9466c3a9d664558ef0faf31073 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 -F src/select.c 61e867a906f140b73baf4ce7a201ad6dcba30820969f5618ee40e9a0d32c6f5f +F src/select.c e1cad752987c476a9d1d053682558caec048ea799ef7ba1b6e38e2266451010d F src/shell.c.in 6a9d8a56700d136addc14d1cd25863b8112e5c12e1877f373750bda476ff6314 F src/sqlite.h.in 41dfd60cf62d4f8756991517c1a9d2ff4977ca04f7798c10b0c16dcf884cbc7e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 960f1b86c3610fa23cb6a267572a97dcf286e77aa0dd3b9b23292ffaa1ea8683 -F src/sqliteInt.h b9f89d79c047df09c1efe2dfe054512cf956780af468daef9d368b319dc68aa7 +F src/sqliteInt.h ca2d5f030547a1677a74bce26338ca70ebd6747b7d500a9edc00146c61ff2376 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -592,7 +592,7 @@ F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392 F src/vtab.c 70188a745dc4e57d26e942681ff4b2912b7c8249ad5de3f60f0677b4337bcfaa F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 41de67424237a3dd097a093057ba454ccfef26c452f6263fecd5591a15dd5a9a +F src/wal.c f53ac2b6ec6f23960471b4402632b8276d8b80c1ac31a28ea08500c56a9cc5c9 F src/wal.h f325a5856b669f5ba449157485915816103857c8574efc746ac55eba3335c5e0 F src/walker.c fb94aadc9099ff9c6506d0a8b88d51266005bcaa265403f3d7caf732a562eb66 F src/where.c 3818e8a736a05d2cb194e64399af707e367fbcc5c251d785804d02eaf121288e @@ -1791,7 +1791,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6f3dd9809fdef7d6195f1f93428a662d5a8c01dba9815daa22d1b94059a2eb43 28a615a2e0f48b0fee3eaf7841ff902e069fa6c221df6ad9a57b8709c88561fb -R c73a7474a8ba36fb4e79620530d56018 +P 76608f750ab13c0a165def9672759fee43cf4e9895df3bfa21765e08358b07a0 +Q +8b1fc4b9f3e743bffcfbbb8e26991240441778e10512fb502b8eaec460184296 +R 05d3a3eaadb134c59c94e4d5941647d0 U dan -Z 4a9121244def783d11e1880a226f98cc +Z 45ce6d06a747e646527aeb46f6440480 diff --git a/manifest.uuid b/manifest.uuid index 04bacb7327..ade3e7b765 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -76608f750ab13c0a165def9672759fee43cf4e9895df3bfa21765e08358b07a0 \ No newline at end of file +81e626f4a44be75425cf916ec61b6b36df0907ebac4adbf6786f87ad4f3a0674 \ No newline at end of file diff --git a/src/func.c b/src/func.c index 0504a8f026..d8233835a2 100644 --- a/src/func.c +++ b/src/func.c @@ -485,8 +485,9 @@ static void randomFunc( sqlite3_value **NotUsed2 ){ sqlite_int64 r; + sqlite3 *db = sqlite3_context_db_handle(context); UNUSED_PARAMETER2(NotUsed, NotUsed2); - sqlite3_randomness(sizeof(r), &r); + sqlite3FastRandomness(&db->sPrng, sizeof(r), &r); if( r<0 ){ /* We need to prevent a random number of 0x8000000000000000 ** (or -9223372036854775808) since when you do abs() of that @@ -512,6 +513,7 @@ static void randomBlob( ){ int n; unsigned char *p; + sqlite3 *db = sqlite3_context_db_handle(context); assert( argc==1 ); UNUSED_PARAMETER(argc); n = sqlite3_value_int(argv[0]); @@ -520,7 +522,7 @@ static void randomBlob( } p = contextMalloc(context, n); if( p ){ - sqlite3_randomness(n, p); + sqlite3FastRandomness(&db->sPrng, n, p); sqlite3_result_blob(context, (char*)p, n, sqlite3_free); } } diff --git a/src/main.c b/src/main.c index 19831abfc7..38b056d93b 100644 --- a/src/main.c +++ b/src/main.c @@ -3023,7 +3023,7 @@ static int openDatabase( db->magic = SQLITE_MAGIC_BUSY; db->aDb = db->aDbStatic; db->lookaside.bDisable = 1; - + sqlite3FastPrngInit(&db->sPrng); assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = SQLITE_DEFAULT_WORKER_THREADS; diff --git a/src/random.c b/src/random.c index d4ae77c435..dd971ab655 100644 --- a/src/random.c +++ b/src/random.c @@ -106,6 +106,28 @@ void sqlite3_randomness(int N, void *pBuf){ sqlite3_mutex_leave(mutex); } +/* +** Initialize a fast PRNG. A Fast PRNG is called "fast" because it does +** not need a mutex to operate, though it does use a mutex to initialize. +** The quality of the randomness is not as good as the global PRNG. +*/ +void sqlite3FastPrngInit(FastPrng *pPrng){ + sqlite3_randomness(sizeof(*pPrng), pPrng); + pPrng->x |= 1; +} + +/* +** Generate N bytes of pseudo-randomness using a FastPrng +*/ +void sqlite3FastRandomness(FastPrng *pPrng, int N, void *P){ + unsigned char *pOut = (unsigned char*)P; + while( N-->0 ){ + pPrng->x = ((pPrng->x)>>1) ^ ((1+~((pPrng->x)&1)) & 0xd0000001); + pPrng->y = (pPrng->y)*1103515245 + 12345; + *(pOut++) = (pPrng->x ^ pPrng->y) & 0xff; + } +} + #ifndef SQLITE_UNTESTABLE /* ** For testing purposes, we sometimes want to preserve the state of diff --git a/src/select.c b/src/select.c index c60ff27001..5cd1630785 100644 --- a/src/select.c +++ b/src/select.c @@ -1993,7 +1993,7 @@ int sqlite3ColumnsFromExprList( if( zName[j]==':' ) nName = j; } zName = sqlite3MPrintf(db, "%.*z:%u", nName, zName, ++cnt); - if( cnt>3 ) sqlite3_randomness(sizeof(cnt), &cnt); + if( cnt>3 ) sqlite3FastRandomness(&db->sPrng, sizeof(cnt), &cnt); } pCol->zName = zName; sqlite3ColumnPropertiesFromName(0, pCol); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 12a9f38867..410cc74c52 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1089,6 +1089,7 @@ typedef struct Parse Parse; typedef struct PreUpdate PreUpdate; typedef struct PrintfArguments PrintfArguments; typedef struct RenameToken RenameToken; +typedef struct FastPrng FastPrng; typedef struct RowSet RowSet; typedef struct Savepoint Savepoint; typedef struct Select Select; @@ -1189,6 +1190,14 @@ typedef int VList; # define SQLITE_DEFAULT_WAL_SYNCHRONOUS SQLITE_DEFAULT_SYNCHRONOUS #endif +/* +** State of a simple PRNG used for the per-connection and per-pager +** pseudo-random number generators. +*/ +struct FastPrng { + unsigned int x, y; +}; + /* ** Each database file to be accessed by the system is an instance ** of the following structure. There are normally two of these structures @@ -1398,6 +1407,7 @@ struct sqlite3 { u8 nSqlExec; /* Number of pending OP_SqlExec opcodes */ int nextPagesize; /* Pagesize after VACUUM if >0 */ u32 magic; /* Magic number for detect library misuse */ + FastPrng sPrng; /* State of the per-connection PRNG */ int nChange; /* Value returned by sqlite3_changes() */ int nTotalChange; /* Value returned by sqlite3_total_changes() */ int aLimit[SQLITE_N_LIMIT]; /* Limits */ @@ -3997,6 +4007,8 @@ Vdbe *sqlite3GetVdbe(Parse*); void sqlite3PrngSaveState(void); void sqlite3PrngRestoreState(void); #endif +void sqlite3FastPrngInit(FastPrng*); +void sqlite3FastRandomness(FastPrng*, int N, void *P); void sqlite3RollbackAll(sqlite3*,int); void sqlite3CodeVerifySchema(Parse*, int); void sqlite3CodeVerifyNamedSchema(Parse*, const char *zDb); diff --git a/src/wal.c b/src/wal.c index 031389ea6f..c67bd9dcac 100644 --- a/src/wal.c +++ b/src/wal.c @@ -474,6 +474,7 @@ struct Wal { u32 nPriorFrame; /* For sqlite3WalInfo() */ const char *zWalName; /* Name of WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ + FastPrng sPrng; /* Random number generator */ #ifdef SQLITE_DEBUG u8 lockError; /* True if a locking error has occurred */ #endif @@ -1380,6 +1381,7 @@ int sqlite3WalOpen( pRet->syncHeader = 1; pRet->padToSectorBoundary = 1; pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); + sqlite3FastPrngInit(&pRet->sPrng); /* Open file handle on the write-ahead log file. */ flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); @@ -1926,7 +1928,7 @@ static int walCheckpoint( rc = SQLITE_BUSY; }else if( eMode>=SQLITE_CHECKPOINT_RESTART ){ u32 salt1; - sqlite3_randomness(4, &salt1); + sqlite3FastRandomness(&pWal->sPrng, 4, &salt1); assert( pInfo->nBackfill==pWal->hdr.mxFrame ); rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); if( rc==SQLITE_OK ){ @@ -3373,7 +3375,7 @@ static int walRestartLog(Wal *pWal){ assert( pInfo->nBackfill==pWal->hdr.mxFrame ); if( pInfo->nBackfill>0 ){ u32 salt1; - sqlite3_randomness(4, &salt1); + sqlite3FastRandomness(&pWal->sPrng, 4, &salt1); rc = walLockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); if( rc==SQLITE_OK ){ /* If all readers are using WAL_READ_LOCK(0) (in other words if no @@ -3589,7 +3591,7 @@ int sqlite3WalFrames( sqlite3Put4byte(&aWalHdr[4], WAL_MAX_VERSION); sqlite3Put4byte(&aWalHdr[8], szPage); sqlite3Put4byte(&aWalHdr[12], pWal->nCkpt); - if( pWal->nCkpt==0 ) sqlite3_randomness(8, pWal->hdr.aSalt); + if( pWal->nCkpt==0 ) sqlite3FastRandomness(&pWal->sPrng, 8, pWal->hdr.aSalt); memcpy(&aWalHdr[16], pWal->hdr.aSalt, 8); walChecksumBytes(1, aWalHdr, WAL_HDRSIZE-2*4, 0, aCksum); sqlite3Put4byte(&aWalHdr[24], aCksum[0]); From 816fc3595655e0f7f543c2d1749f86548ebaabf1 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 3 Dec 2018 18:13:46 +0000 Subject: [PATCH 092/179] Increase a timeout in test file walprotocol2.test. To account for unix builds without HAVE_USLEEP. FossilOrigin-Name: 480be916c840e9e28010c22cf29a8396502c9e6387f31f750260c6290f58e9a1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/walprotocol2.test | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 1531e9dadf..adbd28acea 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Sync\sthis\sbranch\swith\sthe\slatest\strunk. -D 2018-12-01T20:14:06.719 +C Increase\sa\stimeout\sin\stest\sfile\swalprotocol2.test.\sTo\saccount\sfor\sunix\sbuilds\nwithout\sHAVE_USLEEP. +D 2018-12-03T18:13:46.107 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -1619,7 +1619,7 @@ F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496 F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test a112aba0b79e3adeaa485fed09484b32c654e97df58e454aa8489ac2cd57bf84 -F test/walprotocol2.test 7d3b6b4bf0b12f8007121b1e6ef714bc99101fb3b48e46371df1db868eebc131 +F test/walprotocol2.test 7e4bedd5ee83607e2928ac438bf7332a396b980d3e02aa0746509ce11ad1f13c F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20 F test/walro2.test 0e79dd15cbdb4f482c01ea248373669c732414a726b357d04846a816afafb768 F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68 @@ -1782,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 63955442304052f5adddd05ccaeebe87ddc5e25af695702793518f015b4f0a87 bf8c1b2b7a5960c282e543b9c293686dccff272512d08865f4600fb58238b4f9 -R 434605294285e3da7c3ee68026e7ade7 +P 7a44fa5a350a3f19b8e9f5196d22535788885f8c0e849572202bf64a055ddc2d +R 8dcc78dd7464cc5597695c90813a01e4 U dan -Z 6329688d1e7210750fc075a6d45758a7 +Z 7d2b6bedf7bcc33a553c67266fb7675b diff --git a/manifest.uuid b/manifest.uuid index bb767038c1..2987cabc9a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7a44fa5a350a3f19b8e9f5196d22535788885f8c0e849572202bf64a055ddc2d \ No newline at end of file +480be916c840e9e28010c22cf29a8396502c9e6387f31f750260c6290f58e9a1 \ No newline at end of file diff --git a/test/walprotocol2.test b/test/walprotocol2.test index 0792c9aae0..389ccb5528 100644 --- a/test/walprotocol2.test +++ b/test/walprotocol2.test @@ -85,7 +85,7 @@ proc lock_callback {method filename handle lock} { db2 eval { INSERT INTO x VALUES('x') } } } -db timeout 10 +db timeout 1100 do_catchsql_test 2.4 { BEGIN EXCLUSIVE; } {0 {}} From 51883dfc8bcbba9f409d2d1efe8cadbd501000c3 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 3 Dec 2018 19:29:37 +0000 Subject: [PATCH 093/179] Cherrypick a couple of fixes from begin-concurrent-pnu into this branch. The differences between the two branches are now that this one does not have "PRAGMA noop_update" or the mutex-free PRNG. FossilOrigin-Name: a56506b9387a067ef259504d127694ad20223f4b08781d1676ff7f5fdd9443d8 --- manifest | 19 +++++++++----- manifest.uuid | 2 +- src/btree.c | 50 +++++++++++++++++++++++++++++++++--- test/concurrent6.test | 60 +++++++++++++++++++++++++++++++++++++++++++ test/concurrent7.test | 52 +++++++++++++++++++++++++++++++++++++ 5 files changed, 172 insertions(+), 11 deletions(-) create mode 100644 test/concurrent6.test create mode 100644 test/concurrent7.test diff --git a/manifest b/manifest index eadf9a7db7..b00fa69f09 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Bring\sup\sto\sdate\swith\sversion\s3.26.0. -D 2018-12-03T18:15:52.710 +C Cherrypick\sa\scouple\sof\sfixes\sfrom\sbegin-concurrent-pnu\sinto\sthis\sbranch.\sThe\ndifferences\sbetween\sthe\stwo\sbranches\sare\snow\sthat\sthis\sone\sdoes\snot\shave\n"PRAGMA\snoop_update"\sor\sthe\smutex-free\sPRNG. +D 2018-12-03T19:29:37.766 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -451,7 +451,7 @@ F src/auth.c 0fac71038875693a937e506bceb492c5f136dd7b1249fbd4ae70b4e8da14f9df F src/backup.c 78d3cecfbe28230a3a9a1793e2ead609f469be43e8f486ca996006be551857ab F src/bitvec.c 8433d9e98dd6f2ea3286e0d2fe5d65de1bfc18a706486eb2026b01be066b5806 F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 -F src/btree.c 4ad4c92bbd327b72cc7b3444de6b6d0f416e29ef0b21b2c3ec16165ba4070804 +F src/btree.c 4a2184be69d491d4b0228d4e397d67cb0802bbec06e7615b485ea1af69a131f6 F src/btree.h 1ed41c71481a1196a520064f2282bc13d768bbd8ae2850e319a3048f8ee7cb3d F src/btreeInt.h 6c65e6c96f561596f6870c79a64d4706af81613881d7947e3f063e923f14115f F src/build.c 5e04fb8528a4a915ed9af94b5cd068e53f2cfa9824d37464c4059279ea9bc8a0 @@ -735,6 +735,8 @@ F test/concurrent2.test 9dfbeb0a323733fe1d13443371734bb94a674dbf777f464365475903 F test/concurrent3.test 530671ac706f6a1d0f4992dbdd33a86408330d03cd90fb9e82ecb1b27f5fd081 F test/concurrent4.test e0b12cd467137e50259df3b4f837507e82aaa07c35941c88664dc8ed1d089c44 F test/concurrent5.test 0c16cbf7446af162a14e6def30445e94016064eb994e5aa4ebb2bebc59554176 +F test/concurrent6.test a7860e9ca13bb5fb76bcf41c5524fbfa9c37e6e258ecf84ffb5748a272488c67 +F test/concurrent7.test b96fa5c4cfdf8d5c0bc66b6934214500bad0260884a736f054ccc76e81aae85d F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c F test/conflict3.test a83db76a6c3503b2fa057c7bfb08c318d8a422202d8bc5b86226e078e5b49ff9 @@ -1790,7 +1792,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 28a615a2e0f48b0fee3eaf7841ff902e069fa6c221df6ad9a57b8709c88561fb bf8c1b2b7a5960c282e543b9c293686dccff272512d08865f4600fb58238b4f9 -R ad7ac98b07a5e19a9d448dcba578a20a -U drh -Z e005195a2c11998dc5b1c6dec19b9447 +P f0ddb358cc68e5ec6d9e758893ab3da058a3b2e705124a7449279c992e672a4a +Q +50c8952c92b9f0c61935fb0df04ed1426d9e266a812071b7bf5b0215c5552757 +Q +570233716032f258b878d52c4d5a47e07292d66fa84e3a85c0388ec15efee625 +Q +dc0fc2aa7cbefeb5f0ba8c992fd3e9adcfb5a4d61e2321c1bd93f4d36ba9aafc +R 0b6ca72792de50021b48db0500a10b60 +U dan +Z e7051457303763dcda2174967ac468b5 diff --git a/manifest.uuid b/manifest.uuid index d0b8f54408..13eb6ed24f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f0ddb358cc68e5ec6d9e758893ab3da058a3b2e705124a7449279c992e672a4a \ No newline at end of file +a56506b9387a067ef259504d127694ad20223f4b08781d1676ff7f5fdd9443d8 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 828d63c180..f4f7365b92 100644 --- a/src/btree.c +++ b/src/btree.c @@ -656,11 +656,48 @@ static void btreePtrmapDelete(BtShared *pBt){ pBt->pMap = 0; } } + +/* +** Check that the pointer-map does not contain any entries with a parent +** page of 0. Call sqlite3_log() multiple times to output the entire +** data structure if it does. +*/ +static void btreePtrmapCheck(BtShared *pBt, Pgno nPage){ + Pgno i; + int bProblem = 0; + BtreePtrmap *p = pBt->pMap; + + for(i=p->iFirst; i<=nPage; i++){ + PtrmapEntry *pEntry = &p->aPtr[i-p->iFirst]; + if( pEntry->eType==PTRMAP_OVERFLOW1 + || pEntry->eType==PTRMAP_OVERFLOW2 + || pEntry->eType==PTRMAP_BTREE + ){ + if( pEntry->parent==0 ){ + bProblem = 1; + break; + } + } + } + + if( bProblem ){ + for(i=p->iFirst; i<=nPage; i++){ + PtrmapEntry *pEntry = &p->aPtr[i-p->iFirst]; + sqlite3_log(SQLITE_CORRUPT, + "btreePtrmapCheck: pgno=%d eType=%d parent=%d", + (int)i, (int)pEntry->eType, (int)pEntry->parent + ); + } + abort(); + } +} + #else /* SQLITE_OMIT_CONCURRENT */ # define btreePtrmapAllocate(x) SQLITE_OK # define btreePtrmapDelete(x) # define btreePtrmapBegin(x,y) SQLITE_OK # define btreePtrmapEnd(x,y,z) +# define btreePtrmapCheck(y,z) #endif /* SQLITE_OMIT_CONCURRENT */ static void releasePage(MemPage *pPage); /* Forward reference */ @@ -4152,7 +4189,10 @@ static int btreeRelocateRange( if( pEntry->eType==PTRMAP_FREEPAGE ){ Pgno dummy; rc = allocateBtreePage(pBt, &pFree, &dummy, iPg, BTALLOC_EXACT); - releasePage(pFree); + if( pFree ){ + assert( sqlite3PagerPageRefcount(pFree->pDbPage)==1 ); + sqlite3PcacheDrop(pFree->pDbPage); + } assert( rc!=SQLITE_OK || dummy==iPg ); }else if( pnCurrent ){ btreeGetPage(pBt, iPg, &pPg, 0); @@ -4210,6 +4250,8 @@ static int btreeFixUnlocked(Btree *p){ Pgno iHTrunk = get4byte(&p1[32]); u32 nHFree = get4byte(&p1[36]); + btreePtrmapCheck(pBt, nPage); + /* Attach the head database free list to the end of the current ** transactions free-list (if any). */ if( iTrunk!=0 ){ @@ -4236,10 +4278,12 @@ static int btreeFixUnlocked(Btree *p){ /* The current transaction allocated pages pMap->iFirst through ** nPage (inclusive) at the end of the database file. Meanwhile, ** other transactions have allocated (iFirst..nHPage). So move - ** pages (iFirst..MIN(nPage,nHPage)) to (MAX(nPage,nHPage)+1). */ + ** pages (iFirst..MIN(nPage,nHPage)) to (MAX(nPage,nHPage)+1). */ Pgno iLast = MIN(nPage, nHPage); /* Last page to move */ Pgno nCurrent; /* Current size of db */ + nCurrent = MAX(nPage, nHPage); + pBt->nPage = nCurrent; rc = btreeRelocateRange(pBt, pMap->iFirst, iLast, &nCurrent); /* There are now no collisions with the snapshot at the head of the @@ -6229,7 +6273,7 @@ static int allocateBtreePage( ** stores stores the total number of pages on the freelist. */ n = get4byte(&pPage1->aData[36]); testcase( n==mxPage-1 ); - if( ISCONCURRENT==0 && n>=mxPage ){ + if( n>=mxPage ){ return SQLITE_CORRUPT_BKPT; } diff --git a/test/concurrent6.test b/test/concurrent6.test new file mode 100644 index 0000000000..44718b7dbc --- /dev/null +++ b/test/concurrent6.test @@ -0,0 +1,60 @@ +# 2017 May 26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/wal_common.tcl +set ::testprefix concurrent6 + +ifcapable !concurrent { + finish_test + return +} + +sqlite3 db2 test.db + +do_execsql_test 1.0 { + PRAGMA page_size = 1024; + PRAGMA journal_mode = wal; + CREATE TABLE t1(x); + CREATE TABLE t2(x); + CREATE TABLE t3(x); + CREATE TABLE t4(x); + + INSERT INTO t1 VALUES(zeroblob(1500)); +} {wal} + +do_execsql_test -db db2 1.1 { + BEGIN CONCURRENT; + INSERT INTO t3 VALUES(zeroblob(4000)); + DELETE FROM t1; +} + +do_execsql_test 1.2 { + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100) + INSERT INTO t2 SELECT zeroblob(1000) FROM s; + + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100) + INSERT INTO t4 SELECT zeroblob(1000) FROM s; + + DELETE FROM t4; +} + +do_execsql_test -db db2 1.3 { + COMMIT; +} + + +finish_test + diff --git a/test/concurrent7.test b/test/concurrent7.test new file mode 100644 index 0000000000..871e428031 --- /dev/null +++ b/test/concurrent7.test @@ -0,0 +1,52 @@ +# 2018 Jan 5 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix concurrent7 + +sqlite3 db2 test.db + +do_execsql_test 1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x); + CREATE TABLE t2(x); +} {wal} + +do_execsql_test -db db2 2 { + SELECT * FROM t1; +} + +do_execsql_test 3 { + BEGIN CONCURRENT; + INSERT INTO t1 VALUES(randomblob(1500)); + INSERT INTO t1 VALUES(randomblob(1500)); + DELETE FROM t1 WHERE rowid = 1; +} + +do_execsql_test -db db2 4 { + INSERT INTO t2 VALUES(randomblob(1500)); + INSERT INTO t2 VALUES(randomblob(1500)); + INSERT INTO t2 VALUES(randomblob(1500)); + INSERT INTO t2 VALUES(randomblob(1500)); + DELETE FROM t2 WHERE rowid IN (1, 2); +} + +do_execsql_test 5 { + COMMIT; + PRAGMA integrity_check; +} {ok} + +finish_test + + From 834c48c279f8fc016acb0b79e356d3a7f98abc6d Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 3 Dec 2018 20:38:15 +0000 Subject: [PATCH 094/179] Minor change to wal.c on this branch to make it more similar to trunk. FossilOrigin-Name: 6a7af3ead5949c461430c1fa92798dc2bbbc58c8cd504005c5afa38993f0be82 --- manifest | 16 +++++------ manifest.uuid | 2 +- src/wal.c | 73 +++++++++++++++++++-------------------------------- 3 files changed, 35 insertions(+), 56 deletions(-) diff --git a/manifest b/manifest index b00fa69f09..801f2cffb1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Cherrypick\sa\scouple\sof\sfixes\sfrom\sbegin-concurrent-pnu\sinto\sthis\sbranch.\sThe\ndifferences\sbetween\sthe\stwo\sbranches\sare\snow\sthat\sthis\sone\sdoes\snot\shave\n"PRAGMA\snoop_update"\sor\sthe\smutex-free\sPRNG. -D 2018-12-03T19:29:37.766 +C Minor\schange\sto\swal.c\son\sthis\sbranch\sto\smake\sit\smore\ssimilar\sto\strunk. +D 2018-12-03T20:38:15.728 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -592,7 +592,7 @@ F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392 F src/vtab.c 70188a745dc4e57d26e942681ff4b2912b7c8249ad5de3f60f0677b4337bcfaa F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 41de67424237a3dd097a093057ba454ccfef26c452f6263fecd5591a15dd5a9a +F src/wal.c 8a12219e699ed737fa4dd8f0449f1bb5dee765502b48710dbd5be747ea58e65b F src/wal.h f325a5856b669f5ba449157485915816103857c8574efc746ac55eba3335c5e0 F src/walker.c fb94aadc9099ff9c6506d0a8b88d51266005bcaa265403f3d7caf732a562eb66 F src/where.c 3818e8a736a05d2cb194e64399af707e367fbcc5c251d785804d02eaf121288e @@ -1792,10 +1792,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f0ddb358cc68e5ec6d9e758893ab3da058a3b2e705124a7449279c992e672a4a -Q +50c8952c92b9f0c61935fb0df04ed1426d9e266a812071b7bf5b0215c5552757 -Q +570233716032f258b878d52c4d5a47e07292d66fa84e3a85c0388ec15efee625 -Q +dc0fc2aa7cbefeb5f0ba8c992fd3e9adcfb5a4d61e2321c1bd93f4d36ba9aafc -R 0b6ca72792de50021b48db0500a10b60 +P a56506b9387a067ef259504d127694ad20223f4b08781d1676ff7f5fdd9443d8 +R d66e284ab90799b5d6b7d2957b4e83a3 +T +closed 0d12f49feb78a94a1b188e80379b51dfe9bf6c8e60225134e15216192cabed21 U dan -Z e7051457303763dcda2174967ac468b5 +Z 035c0f4c166e3fab14296a764b5a28cd diff --git a/manifest.uuid b/manifest.uuid index 13eb6ed24f..dd8dc8b081 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a56506b9387a067ef259504d127694ad20223f4b08781d1676ff7f5fdd9443d8 \ No newline at end of file +6a7af3ead5949c461430c1fa92798dc2bbbc58c8cd504005c5afa38993f0be82 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 031389ea6f..673a159b19 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2854,19 +2854,37 @@ void sqlite3WalEndReadTransaction(Wal *pWal){ } /* -** Search the hash tables for an entry matching page number pgno. Ignore -** any entries that lie after frame iLast within the wal file. +** Search the wal file for page pgno. If found, set *piRead to the frame that +** contains the page. Otherwise, if pgno is not in the wal file, set *piRead +** to zero. +** +** Return SQLITE_OK if successful, or an error code if an error occurs. If an +** error does occur, the final value of *piRead is undefined. */ -static int walFindFrame( - Wal *pWal, - Pgno pgno, - u32 iLast, - u32 *piRead +int sqlite3WalFindFrame( + Wal *pWal, /* WAL handle */ + Pgno pgno, /* Database page number to read data for */ + u32 *piRead /* OUT: Frame number (or zero) */ ){ + u32 iRead = 0; /* If !=0, WAL frame to return data from */ + u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ int iHash; /* Used to loop through N hash tables */ - u32 iRead = 0; int iMinHash; + /* This routine is only be called from within a read transaction. */ + assert( pWal->readLock>=0 || pWal->lockError ); + + /* If the "last page" field of the wal-index header snapshot is 0, then + ** no data will be read from the wal under any circumstances. Return early + ** in this case as an optimization. Likewise, if pWal->readLock==0, + ** then the WAL is ignored by the reader so return early, as if the + ** WAL were empty. + */ + if( iLast==0 || (pWal->readLock==0 && pWal->bShmUnreliable==0) ){ + *piRead = 0; + return SQLITE_OK; + } + /* Each iteration of the following for() loop searches one ** hash table (each hash table indexes up to HASHTABLE_NPAGE frames). ** @@ -2917,43 +2935,6 @@ static int walFindFrame( if( iRead ) break; } - *piRead = iRead; - return SQLITE_OK; -} - -/* -** Search the wal file for page pgno. If found, set *piRead to the frame that -** contains the page. Otherwise, if pgno is not in the wal file, set *piRead -** to zero. -** -** Return SQLITE_OK if successful, or an error code if an error occurs. If an -** error does occur, the final value of *piRead is undefined. -*/ -int sqlite3WalFindFrame( - Wal *pWal, /* WAL handle */ - Pgno pgno, /* Database page number to read data for */ - u32 *piRead /* OUT: Frame number (or zero) */ -){ - u32 iRead = 0; /* If !=0, WAL frame to return data from */ - u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ - int rc; - - /* This routine is only be called from within a read transaction. */ - assert( pWal->readLock>=0 || pWal->lockError ); - - /* If the "last page" field of the wal-index header snapshot is 0, then - ** no data will be read from the wal under any circumstances. Return early - ** in this case as an optimization. Likewise, if pWal->readLock==0, - ** then the WAL is ignored by the reader so return early, as if the - ** WAL were empty. - */ - if( iLast==0 || (pWal->readLock==0 && pWal->bShmUnreliable==0) ){ - *piRead = 0; - return SQLITE_OK; - } - - rc = walFindFrame(pWal, pgno, iLast, &iRead); - #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT /* If expensive assert() statements are available, do a linear search ** of the wal-index file content. Make sure the results agree with the @@ -2973,7 +2954,7 @@ int sqlite3WalFindFrame( #endif *piRead = iRead; - return rc; + return SQLITE_OK; } /* From 50232dd821f419f3f0a667f0f9699e714e567641 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 4 Dec 2018 13:51:43 +0000 Subject: [PATCH 095/179] Fix a problem with SQLITE_ENABLE_EXPENSIVE_ASSERT builds on this branch. FossilOrigin-Name: ddb4a6fbf8619db058b5eb8fcd687084ed4b65a6f69810357e324158257a911f --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/wal.c | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 801f2cffb1..6115220691 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\schange\sto\swal.c\son\sthis\sbranch\sto\smake\sit\smore\ssimilar\sto\strunk. -D 2018-12-03T20:38:15.728 +C Fix\sa\sproblem\swith\sSQLITE_ENABLE_EXPENSIVE_ASSERT\sbuilds\son\sthis\sbranch. +D 2018-12-04T13:51:43.293 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -592,7 +592,7 @@ F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392 F src/vtab.c 70188a745dc4e57d26e942681ff4b2912b7c8249ad5de3f60f0677b4337bcfaa F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 8a12219e699ed737fa4dd8f0449f1bb5dee765502b48710dbd5be747ea58e65b +F src/wal.c 6675d84d7bc5d04879e487d79248fb79d9bdad46811cc5ed5ef6233bdf1b70a1 F src/wal.h f325a5856b669f5ba449157485915816103857c8574efc746ac55eba3335c5e0 F src/walker.c fb94aadc9099ff9c6506d0a8b88d51266005bcaa265403f3d7caf732a562eb66 F src/where.c 3818e8a736a05d2cb194e64399af707e367fbcc5c251d785804d02eaf121288e @@ -1792,8 +1792,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a56506b9387a067ef259504d127694ad20223f4b08781d1676ff7f5fdd9443d8 -R d66e284ab90799b5d6b7d2957b4e83a3 -T +closed 0d12f49feb78a94a1b188e80379b51dfe9bf6c8e60225134e15216192cabed21 +P 6a7af3ead5949c461430c1fa92798dc2bbbc58c8cd504005c5afa38993f0be82 +R d464b4876361195a5c5bd9cc28bc9bdf U dan -Z 035c0f4c166e3fab14296a764b5a28cd +Z 0763a5ad9b6082127adcd5e3f801575e diff --git a/manifest.uuid b/manifest.uuid index dd8dc8b081..f3449cc2a0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6a7af3ead5949c461430c1fa92798dc2bbbc58c8cd504005c5afa38993f0be82 \ No newline at end of file +ddb4a6fbf8619db058b5eb8fcd687084ed4b65a6f69810357e324158257a911f \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 673a159b19..89f475d4d1 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2939,7 +2939,7 @@ int sqlite3WalFindFrame( /* If expensive assert() statements are available, do a linear search ** of the wal-index file content. Make sure the results agree with the ** result obtained using the hash indexes above. */ - if( rc==SQLITE_OK ){ + { u32 iRead2 = 0; u32 iTest; assert( pWal->bShmUnreliable || pWal->minFrame>0 ); From 7a13e692be60a25104daeb2e921aa2a0a82cd244 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 4 Dec 2018 19:41:07 +0000 Subject: [PATCH 096/179] First attempt at making features work together. Only the most minimal testing so far. FossilOrigin-Name: fd707001f0afb1cf32cfeeda3ec7b5622eb49ddedf8fec1a7aa4c8841c77c37a --- manifest | 16 ++-- manifest.uuid | 2 +- src/wal.c | 180 +++++++++++++++++++++++++-------------- test/wal2concurrent.test | 84 ++++++++++++++++++ 4 files changed, 207 insertions(+), 75 deletions(-) create mode 100644 test/wal2concurrent.test diff --git a/manifest b/manifest index dd9771b7e3..dc2ae521ba 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\swal2\sand\sbegin-concurrent\scode.\sBoth\sfeatures\swork,\sbut\snot\sat\sthe\nsame\stime. -D 2018-12-03T20:49:34.015 +C First\sattempt\sat\smaking\sfeatures\swork\stogether.\sOnly\sthe\smost\sminimal\stesting\nso\sfar. +D 2018-12-04T19:41:07.389 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -593,7 +593,7 @@ F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392 F src/vtab.c 70188a745dc4e57d26e942681ff4b2912b7c8249ad5de3f60f0677b4337bcfaa F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 88e424b4de9cb836166a967b5b8a6de7f90cdd493dd1d04a1b855fa58dbe7212 +F src/wal.c 4b7ad0e623e01ede1ecb2cae24548c29e51ce5a21b2fd91af26b4bb447b72cc4 F src/wal.h b42fc8081cd1765d4d4dd99b33f2db2f71128f4e25ff8c08d1a346f5af62e27a F src/walker.c fb94aadc9099ff9c6506d0a8b88d51266005bcaa265403f3d7caf732a562eb66 F src/where.c 3818e8a736a05d2cb194e64399af707e367fbcc5c251d785804d02eaf121288e @@ -1606,6 +1606,7 @@ F test/vtab_err.test dcc8b7b9cb67522b3fe7a272c73856829dae4ab7fdb30399aea1b6981bd F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 F test/wal2.test a225bafac35a47765b890bacdeb57e5e81039f21cc18a1e8ce88eb76e56b843c +F test/wal2concurrent.test d0af64c3b113993553acfeeb64d207fb6d924e498fd953dbf31040a566ec10df F test/wal2rewrite.test 6ca6f631ffcf871240beab5f02608913fd075c6d0d31310b026c8383c65c9f9c F test/wal2simple.test 8c9dfb8f1bca01a0deb57f7074cdb83865c2292e89b13f7a51a1c160dca3f5f4 F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2 @@ -1795,10 +1796,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6a7af3ead5949c461430c1fa92798dc2bbbc58c8cd504005c5afa38993f0be82 480be916c840e9e28010c22cf29a8396502c9e6387f31f750260c6290f58e9a1 -R 41c3c1de1e62db08c33d5cf08c11006b -T *branch * begin-concurrent-wal2 -T *sym-begin-concurrent-wal2 * -T -sym-begin-concurrent * +P b7281a1caa574870a071bea3e96b1d8210c28c17f9094449b3ce1a42b311e6a1 +R db024f31bd5bbdadf08ded59e2f54e6f U dan -Z f5d7b930a621f73c3ef64e6a734bda83 +Z abc3921392e51c645845b41c65979599 diff --git a/manifest.uuid b/manifest.uuid index a398ac8e87..e66f70d089 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b7281a1caa574870a071bea3e96b1d8210c28c17f9094449b3ce1a42b311e6a1 \ No newline at end of file +fd707001f0afb1cf32cfeeda3ec7b5622eb49ddedf8fec1a7aa4c8841c77c37a \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 5b50c99dfa..3c0e73967d 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2850,7 +2850,7 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){ ** even if some external agent does a "chmod" to make the shared-memory ** writable by us, until sqlite3OsShmUnmap() has been called. ** This is a requirement on the VFS implementation. - */ + */ rc = sqlite3OsShmMap(pWal->pDbFd, 0, WALINDEX_PGSZ, 0, &pDummy); assert( rc!=SQLITE_OK ); /* SQLITE_OK not possible for read-only connection */ if( rc!=SQLITE_READONLY_CANTINIT ){ @@ -3531,23 +3531,32 @@ int sqlite3WalFindFrame( /* This routine is only be called from within a read transaction. */ assert( pWal->readLock!=WAL_LOCK_NONE ); - /* If this is a wal2 system, the client must have a partial-wal lock - ** on wal file iApp. Or if it is a wal system, iApp==0 must be true. */ - assert( bWal2==0 || iApp==1 - || pWal->readLock==WAL_LOCK_PART1 || pWal->readLock==WAL_LOCK_PART1_FULL2 - ); - assert( bWal2==0 || iApp==0 - || pWal->readLock==WAL_LOCK_PART2 || pWal->readLock==WAL_LOCK_PART2_FULL1 - ); - assert( bWal2 || iApp==0 ); + /* If this is a regular wal system, then iApp must be set to 0 (there is + ** only one wal file, after all). Or, if this is a wal2 system and the + ** write-lock is not held, the client must have a partial-wal lock on wal + ** file iApp. This is not always true if the write-lock is held and this + ** function is being called after WalLockForCommit() as part of committing + ** a CONCURRENT transaction. */ +#ifdef SQLITE_DEBUG + if( bWal2 ){ + if( pWal->writeLock==0 ){ + int l = pWal->readLock; + assert( iApp==1 || l==WAL_LOCK_PART1 || l==WAL_LOCK_PART1_FULL2 ); + assert( iApp==0 || l==WAL_LOCK_PART2 || l==WAL_LOCK_PART2_FULL1 ); + } + }else{ + assert( iApp==0 ); + } +#endif /* Return early if read-lock 0 is held. */ if( (pWal->readLock==0 && pWal->bShmUnreliable==0) ){ + assert( !bWal2 ); *piRead = 0; return SQLITE_OK; } - /* Search the wal file that the client holds a partial lock on first */ + /* Search the wal file that the client holds a partial lock on first. */ rc = walSearchWal(pWal, iApp, pgno, &iRead); /* If the requested page was not found, no error has occured, and @@ -3556,6 +3565,10 @@ int sqlite3WalFindFrame( if( rc==SQLITE_OK && bWal2 && iRead==0 && ( pWal->readLock==WAL_LOCK_PART1_FULL2 || pWal->readLock==WAL_LOCK_PART2_FULL1 +#ifndef SQLITE_OMIT_CONCURRENT + || (pWal->readLock==WAL_LOCK_PART1 && iApp==1) + || (pWal->readLock==WAL_LOCK_PART2 && iApp==0) +#endif )){ rc = walSearchWal(pWal, !iApp, pgno, &iRead); } @@ -3784,67 +3797,104 @@ int sqlite3WalLockForCommit( ** was opened. */ rc = SQLITE_BUSY_SNAPSHOT; }else if( memcmp(&pWal->hdr, (void*)&head, sizeof(WalIndexHdr))!=0 ){ + int bWal2 = isWalMode2(pWal); int iHash; int iLastHash = walFramePage(head.mxFrame); - u32 iFirst = pWal->hdr.mxFrame+1; /* First wal frame to check */ - if( memcmp(pWal->hdr.aSalt, (u32*)head.aSalt, sizeof(u32)*2) ){ - assert( pWal->readLock==0 ); - iFirst = 1; - } - for(iHash=walFramePage(iFirst); iHash<=iLastHash; iHash++){ - WalHashLoc sLoc; + int nLoop = 1+(bWal2 && walidxGetFile(&head)!=walidxGetFile(&pWal->hdr)); + int iLoop; + + + assert( nLoop==1 || nLoop==2 ); + for(iLoop=0; iLoophdr.mxFrame (which will be + ** set to the size of the old, now overwritten, wal file). This + ** doesn't come up in wal2 mode, as in wal2 mode the client always + ** has a PART lock on one of the wal files, preventing it from being + ** checkpointed or overwritten. */ + iFirst = pWal->hdr.mxFrame+1; + if( memcmp(pWal->hdr.aSalt, (u32*)head.aSalt, sizeof(u32)*2) ){ + assert( pWal->readLock==0 ); + iFirst = 1; + } + mxFrame = head.mxFrame; + }else{ + int iA = walidxGetFile(&pWal->hdr); + if( iLoop==0 ){ + iFirst = walExternalEncode(iA, 1+walidxGetMxFrame(&pWal->hdr, iA)); + mxFrame = walExternalEncode(iA, walidxGetMxFrame(&head, iA)); + }else{ + iFirst = walExternalEncode(!iA, 1); + mxFrame = walExternalEncode(!iA, walidxGetMxFrame(&head, !iA)); + } + } + iLastHash = walFramePage(mxFrame); - rc = walHashGet(pWal, iHash, &sLoc); - if( rc==SQLITE_OK ){ - u32 i, iMin, iMax; - assert( head.mxFrame>=sLoc.iZero ); - iMin = (sLoc.iZero >= iFirst) ? 1 : (iFirst - sLoc.iZero); - iMax = (iHash==0) ? HASHTABLE_NPAGE_ONE : HASHTABLE_NPAGE; - if( iMax>(head.mxFrame-sLoc.iZero) ) iMax = (head.mxFrame-sLoc.iZero); - for(i=iMin; rc==SQLITE_OK && i<=iMax; i++){ - PgHdr *pPg; - if( sLoc.aPgno[i]==1 ){ - /* Check that the schema cookie has not been modified. If - ** it has not, the commit can proceed. */ - u8 aNew[4]; - u8 *aOld = &((u8*)pPage1->pData)[40]; - int sz; - i64 iOffset; - sz = pWal->hdr.szPage; - sz = (sz&0xfe00) + ((sz&0x0001)<<16); - iOffset = walFrameOffset(i+sLoc.iZero, sz) + WAL_FRAME_HDRSIZE+40; - rc = sqlite3OsRead(pWal->apWalFd[0], aNew, sizeof(aNew), iOffset); - if( rc==SQLITE_OK && memcmp(aOld, aNew, sizeof(aNew)) ){ + for(iHash=walFramePage(iFirst); iHash<=iLastHash; iHash += (1+bWal2)){ + WalHashLoc sLoc; + + rc = walHashGet(pWal, iHash, &sLoc); + if( rc==SQLITE_OK ){ + u32 i, iMin, iMax; + assert( mxFrame>=sLoc.iZero ); + iMin = (sLoc.iZero >= iFirst) ? 1 : (iFirst - sLoc.iZero); + iMax = (iHash==0) ? HASHTABLE_NPAGE_ONE : HASHTABLE_NPAGE; + if( iMax>(mxFrame-sLoc.iZero) ) iMax = (mxFrame-sLoc.iZero); + for(i=iMin; rc==SQLITE_OK && i<=iMax; i++){ + PgHdr *pPg; + if( sLoc.aPgno[i]==1 ){ + /* Check that the schema cookie has not been modified. If + ** it has not, the commit can proceed. */ + u8 aNew[4]; + u8 *aOld = &((u8*)pPage1->pData)[40]; + int sz; + i64 iOffset; + sz = pWal->hdr.szPage; + sz = (sz&0xfe00) + ((sz&0x0001)<<16); + iOffset = walFrameOffset(i+sLoc.iZero, sz)+WAL_FRAME_HDRSIZE+40; + rc = sqlite3OsRead(pWal->apWalFd[0], aNew,sizeof(aNew),iOffset); + if( rc==SQLITE_OK && memcmp(aOld, aNew, sizeof(aNew)) ){ + rc = SQLITE_BUSY_SNAPSHOT; + } + }else if( sqlite3BitvecTestNotNull(pAllRead, sLoc.aPgno[i]) ){ + *piConflict = sLoc.aPgno[i]; rc = SQLITE_BUSY_SNAPSHOT; - } - }else if( sqlite3BitvecTestNotNull(pAllRead, sLoc.aPgno[i]) ){ - *piConflict = sLoc.aPgno[i]; - rc = SQLITE_BUSY_SNAPSHOT; - }else if( (pPg = sqlite3PagerLookup(pPager, sLoc.aPgno[i])) ){ - /* Page aPgno[i], which is present in the pager cache, has been - ** modified since the current CONCURRENT transaction was started. - ** However it was not read by the current transaction, so is not - ** a conflict. There are two possibilities: (a) the page was - ** allocated at the of the file by the current transaction or - ** (b) was present in the cache at the start of the transaction. - ** - ** For case (a), do nothing. This page will be moved within the - ** database file by the commit code to avoid the conflict. The - ** call to PagerUnref() is to release the reference grabbed by - ** the sqlite3PagerLookup() above. - ** - ** In case (b), drop the page from the cache - otherwise - ** following the snapshot upgrade the cache would be inconsistent - ** with the database as stored on disk. */ - if( sqlite3PagerIswriteable(pPg) ){ - sqlite3PagerUnref(pPg); - }else{ - sqlite3PcacheDrop(pPg); + }else if( (pPg = sqlite3PagerLookup(pPager, sLoc.aPgno[i])) ){ + /* Page aPgno[i], which is present in the pager cache, has been + ** modified since the current CONCURRENT transaction was + ** started. However it was not read by the current + ** transaction, so is not a conflict. There are two + ** possibilities: (a) the page was allocated at the of the file + ** by the current transaction or (b) was present in the cache + ** at the start of the transaction. + ** + ** For case (a), do nothing. This page will be moved within the + ** database file by the commit code to avoid the conflict. The + ** call to PagerUnref() is to release the reference grabbed by + ** the sqlite3PagerLookup() above. + ** + ** In case (b), drop the page from the cache - otherwise + ** following the snapshot upgrade the cache would be + ** inconsistent with the database as stored on disk. */ + if( sqlite3PagerIswriteable(pPg) ){ + sqlite3PagerUnref(pPg); + }else{ + sqlite3PcacheDrop(pPg); + } } } } + if( rc!=SQLITE_OK ) break; } - if( rc!=SQLITE_OK ) break; } } } @@ -3876,6 +3926,7 @@ int sqlite3WalUpgradeSnapshot(Wal *pWal){ ** any reads performed between now and committing the transaction will ** read from the old snapshot - not the one just upgraded to. */ if( pWal->readLock==0 && pWal->hdr.mxFrame!=walCkptInfo(pWal)->nBackfill ){ + assert( isWalMode2(pWal)==0 ); rc = walUpgradeReadlock(pWal); } return rc; @@ -3959,7 +4010,6 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ */ void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){ int iWal = walidxGetFile(&pWal->hdr); - assert( pWal->writeLock ); assert( isWalMode2(pWal) || iWal==0 ); aWalData[0] = walidxGetMxFrame(&pWal->hdr, iWal); aWalData[1] = pWal->hdr.aFrameCksum[0]; diff --git a/test/wal2concurrent.test b/test/wal2concurrent.test new file mode 100644 index 0000000000..db0c8485b9 --- /dev/null +++ b/test/wal2concurrent.test @@ -0,0 +1,84 @@ +# 2015 July 26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set ::testprefix wal2concurrent + +ifcapable !concurrent { + finish_test + return +} + + +#------------------------------------------------------------------------- +# Warm-body test. +# +sqlite3 db2 test.db +do_execsql_test 1.0 { + PRAGMA page_size = 1024; + CREATE TABLE t1(x); + CREATE TABLE t2(y); + PRAGMA journal_size_limit = 5000; + PRAGMA journal_mode = wal2; +} {5000 wal2} + +do_execsql_test 1.1 { + INSERT INTO t1 VALUES(1); + BEGIN CONCURRENT; + INSERT INTO t1 VALUES(2); +} {} + +do_test 1.2 { + execsql { + PRAGMA journal_size_limit = 5000; + INSERT INTO t1 VALUES(3) + } db2 + catchsql { COMMIT } +} {1 {database is locked}} + +do_catchsql_test 1.3 { COMMIT } {1 {database is locked}} +do_catchsql_test 1.4 { ROLLBACK } {0 {}} + +do_test 1.5 { + list [file size test.db-wal] [file size test.db-wal2] +} {2128 0} + +do_execsql_test 1.6 { + BEGIN CONCURRENT; + INSERT INTO t1 VALUES(2); +} {} + +do_test 1.7 { + execsql { INSERT INTO t2 VALUES(randomblob(4000)) } db2 + list [file size test.db-wal] [file size test.db-wal2] +} {7368 0} + +do_test 1.8 { + execsql { + INSERT INTO t2 VALUES(1); + INSERT INTO t1 VALUES(5); + } db2 + list [file size test.db-wal] [file size test.db-wal2] +} {7368 2128} + +do_catchsql_test 1.9 { COMMIT } {1 {database is locked}} +do_catchsql_test 1.10 { ROLLBACK } {0 {}} + +db close +sqlite3 db test.db +do_execsql_test 1.11 { SELECT * FROM t1 } {1 3 5} +do_execsql_test 1.12 { SELECT count(*) FROM t2 } {2} + +finish_test + From 6456b3929c58287efdd0851b3ef14e935c78ade1 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 5 Dec 2018 16:31:02 +0000 Subject: [PATCH 097/179] Fixes for snapshots API on this branch. Also ensure that the snapshots API cannot be used with wal2 mode databases (for now anyhow). FossilOrigin-Name: 19c61ab79458936ff4dfca46cf4d1fb1ab16d7bdb5024f502eb4339ec4eef32c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 13 ++++++++++++- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index adbd28acea..5cf7e8b10d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sa\stimeout\sin\stest\sfile\swalprotocol2.test.\sTo\saccount\sfor\sunix\sbuilds\nwithout\sHAVE_USLEEP. -D 2018-12-03T18:13:46.107 +C Fixes\sfor\ssnapshots\sAPI\son\sthis\sbranch.\sAlso\sensure\sthat\sthe\ssnapshots\sAPI\ncannot\sbe\sused\swith\swal2\smode\sdatabases\s(for\snow\sanyhow). +D 2018-12-05T16:31:02.698 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -589,7 +589,7 @@ F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392 F src/vtab.c 70188a745dc4e57d26e942681ff4b2912b7c8249ad5de3f60f0677b4337bcfaa F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c e5b19ec1ce95882a2ae610d2657b441b079734427757c1a4d069f5f57bb7e9a2 +F src/wal.c 2b9b1bc7eeb819b46b0abe0bf4d9c48a13df0989814bb213da1b480d4b1c8107 F src/wal.h fc6113057f2950fc14631176b748293e216fb385ea8df665e6d259e37f8f7d21 F src/walker.c fb94aadc9099ff9c6506d0a8b88d51266005bcaa265403f3d7caf732a562eb66 F src/where.c 3818e8a736a05d2cb194e64399af707e367fbcc5c251d785804d02eaf121288e @@ -1782,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7a44fa5a350a3f19b8e9f5196d22535788885f8c0e849572202bf64a055ddc2d -R 8dcc78dd7464cc5597695c90813a01e4 +P 480be916c840e9e28010c22cf29a8396502c9e6387f31f750260c6290f58e9a1 +R 869a87f8972d12d94814188a57f87377 U dan -Z 7d2b6bedf7bcc33a553c67266fb7675b +Z 200899075d870fbfb9b7c809d447f2d7 diff --git a/manifest.uuid b/manifest.uuid index 2987cabc9a..0754a48ec2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -480be916c840e9e28010c22cf29a8396502c9e6387f31f750260c6290f58e9a1 \ No newline at end of file +19c61ab79458936ff4dfca46cf4d1fb1ab16d7bdb5024f502eb4339ec4eef32c \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 42001bf2d2..97bb781d79 100644 --- a/src/wal.c +++ b/src/wal.c @@ -3234,6 +3234,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ int sqlite3WalSnapshotRecover(Wal *pWal){ int rc; + /* Snapshots may not be used with wal2 mode databases. */ + if( isWalMode2(pWal) ) return SQLITE_ERROR; + assert( pWal->readLock>=0 ); rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); if( rc==SQLITE_OK ){ @@ -3308,6 +3311,7 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ #ifdef SQLITE_ENABLE_SNAPSHOT int bChanged = 0; WalIndexHdr *pSnapshot = pWal->pSnapshot; + if( pSnapshot && isWalMode2(pWal) ) return SQLITE_ERROR; if( pSnapshot && memcmp(pSnapshot, &pWal->hdr, sizeof(WalIndexHdr))!=0 ){ bChanged = 1; } @@ -4534,9 +4538,12 @@ int sqlite3WalSnapshotGet(Wal *pWal, sqlite3_snapshot **ppSnapshot){ WalIndexHdr *pRet; static const u32 aZero[4] = { 0, 0, 0, 0 }; + /* Snapshots may not be used with wal2 mode databases. */ + if( isWalMode2(pWal) ) return SQLITE_ERROR; + assert( pWal->readLock>=0 && pWal->writeLock==0 ); - if( memcmp(&pWal->hdr.aFrameCksum[0],aZero,16)==0 ){ + if( memcmp(&pWal->hdr.aFrameCksum[0],aZero,8)==0 ){ *ppSnapshot = 0; return SQLITE_ERROR; } @@ -4587,6 +4594,10 @@ int sqlite3_snapshot_cmp(sqlite3_snapshot *p1, sqlite3_snapshot *p2){ */ int sqlite3WalSnapshotCheck(Wal *pWal, sqlite3_snapshot *pSnapshot){ int rc; + + /* Snapshots may not be used with wal2 mode databases. */ + if( isWalMode2(pWal) ) return SQLITE_ERROR; + rc = walLockShared(pWal, WAL_CKPT_LOCK); if( rc==SQLITE_OK ){ WalIndexHdr *pNew = (WalIndexHdr*)pSnapshot; From e05f9225831a41d64127e88ebc2add79eb002b8a Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 5 Dec 2018 17:14:03 +0000 Subject: [PATCH 098/179] Fix a problem causing "PRAGMA journal_mode" to report the wrong journal mode (wal instead of wal2) under some circumstances. FossilOrigin-Name: 1d8d4f689653ce80157740e339f7f1b42479bf90d82b176d8202d0a49f428398 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/pager.c | 4 ++++ src/wal.c | 8 ++++++++ src/wal.h | 4 ++++ 5 files changed, 25 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 5cf7e8b10d..f47ab23d2f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sfor\ssnapshots\sAPI\son\sthis\sbranch.\sAlso\sensure\sthat\sthe\ssnapshots\sAPI\ncannot\sbe\sused\swith\swal2\smode\sdatabases\s(for\snow\sanyhow). -D 2018-12-05T16:31:02.698 +C Fix\sa\sproblem\scausing\s"PRAGMA\sjournal_mode"\sto\sreport\sthe\swrong\sjournal\smode\n(wal\sinstead\sof\swal2)\sunder\ssome\scircumstances. +D 2018-12-05T17:14:03.651 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -494,7 +494,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c 711480e9152f221098ec2b0d4ef94dc798f08af649c34f5cd4dc2bbf40c4f556 F src/os_win.c 85d9e532d0444ab6c16d7431490c2e279e282aa0917b0e988996b1ae0de5c5a0 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c dd88ccf7fa519d6e325a115af28cd528713ea4d33a510d6f202386df002f2e74 +F src/pager.c dc6d0acb3f89b4dcb801c86a0e722c1bde116e058c7216ff74341812a38bbdd9 F src/pager.h 3abf6d65199fd0680b26a047c6167a96a4d6ead7535e02522b79f0fb27a3edec F src/parse.y 6840fe7c0b5eb4dd25ee5d075213bc8255ed4c0678d71bfb6744d0520d91c179 F src/pcache.c 696a01f1a6370c1b50a09c15972bc3bee3333f8fcd1f2da8e9a76b1b062c59ee @@ -589,8 +589,8 @@ F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392 F src/vtab.c 70188a745dc4e57d26e942681ff4b2912b7c8249ad5de3f60f0677b4337bcfaa F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 2b9b1bc7eeb819b46b0abe0bf4d9c48a13df0989814bb213da1b480d4b1c8107 -F src/wal.h fc6113057f2950fc14631176b748293e216fb385ea8df665e6d259e37f8f7d21 +F src/wal.c c01f62932ae8458e77aed8846e819e167a52224c0a13f972c9e7d90354205911 +F src/wal.h d2a69695c84137f76e19a247a342cb02ab0131001b6f58153d94b71195bbd84d F src/walker.c fb94aadc9099ff9c6506d0a8b88d51266005bcaa265403f3d7caf732a562eb66 F src/where.c 3818e8a736a05d2cb194e64399af707e367fbcc5c251d785804d02eaf121288e F src/whereInt.h f125f29fca80890768e0b2caa14f95db74b2dacd3a122a168f97aa7b64d6968f @@ -1782,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 480be916c840e9e28010c22cf29a8396502c9e6387f31f750260c6290f58e9a1 -R 869a87f8972d12d94814188a57f87377 +P 19c61ab79458936ff4dfca46cf4d1fb1ab16d7bdb5024f502eb4339ec4eef32c +R b8125f586e18293d9635394478d15d26 U dan -Z 200899075d870fbfb9b7c809d447f2d7 +Z 00401c2c33715960f0bf723cc059ce5c diff --git a/manifest.uuid b/manifest.uuid index 0754a48ec2..ebdf4ef09b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -19c61ab79458936ff4dfca46cf4d1fb1ab16d7bdb5024f502eb4339ec4eef32c \ No newline at end of file +1d8d4f689653ce80157740e339f7f1b42479bf90d82b176d8202d0a49f428398 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 1dbbd4c631..72abf2f94f 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3258,6 +3258,10 @@ static int pagerBeginReadTransaction(Pager *pPager){ if( rc!=SQLITE_OK || changed ){ pager_reset(pPager); if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0); + assert( pPager->journalMode==PAGER_JOURNALMODE_WAL + || pPager->journalMode==PAGER_JOURNALMODE_WAL2 + ); + pPager->journalMode = sqlite3WalJournalMode(pPager->pWal); } return rc; diff --git a/src/wal.c b/src/wal.c index 97bb781d79..ee5d2e1779 100644 --- a/src/wal.c +++ b/src/wal.c @@ -4641,4 +4641,12 @@ sqlite3_file *sqlite3WalFile(Wal *pWal){ return pWal->apWalFd[0]; } +/* +** Return the journal mode used by this Wal object. +*/ +int sqlite3WalJournalMode(Wal *pWal){ + assert( pWal ); + return (isWalMode2(pWal) ? PAGER_JOURNALMODE_WAL2 : PAGER_JOURNALMODE_WAL); +} + #endif /* #ifndef SQLITE_OMIT_WAL */ diff --git a/src/wal.h b/src/wal.h index 7017e908ed..ccc2888f97 100644 --- a/src/wal.h +++ b/src/wal.h @@ -45,6 +45,7 @@ # define sqlite3WalFramesize(z) 0 # define sqlite3WalFindFrame(x,y,z) 0 # define sqlite3WalFile(x) 0 +# define sqlite3WalJournalMode(x) 0 #else #define WAL_SAVEPOINT_NDATA 4 @@ -146,5 +147,8 @@ int sqlite3WalFramesize(Wal *pWal); /* Return the sqlite3_file object for the WAL file */ sqlite3_file *sqlite3WalFile(Wal *pWal); +/* Return the journal mode (WAL or WAL2) used by this Wal object. */ +int sqlite3WalJournalMode(Wal *pWal); + #endif /* ifndef SQLITE_OMIT_WAL */ #endif /* SQLITE_WAL_H */ From c30d437376a6ac366319ec01716700bc45414d9d Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 5 Dec 2018 17:31:24 +0000 Subject: [PATCH 099/179] Fix a test script problem on this branch. FossilOrigin-Name: 285d1c5904dd607457c6b5ff7be3d3191a90ca8b1fb0837327663c9910f978ac --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/savepoint.test | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index f47ab23d2f..5b4bda667c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\scausing\s"PRAGMA\sjournal_mode"\sto\sreport\sthe\swrong\sjournal\smode\n(wal\sinstead\sof\swal2)\sunder\ssome\scircumstances. -D 2018-12-05T17:14:03.651 +C Fix\sa\stest\sscript\sproblem\son\sthis\sbranch. +D 2018-12-05T17:31:24.866 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -1227,7 +1227,7 @@ F test/rowvalue9.test d8dd2c6ecac432dadaa79e41dc2434f007be1b6b F test/rowvaluefault.test 7cd9ccc6c2fbdd881672984087aad0491bb75504 F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 F test/run-wordcount.sh 891e89c4c2d16e629cd45951d4ed899ad12afc09 -F test/savepoint.test 69c56b891ce0ff28f1376b5516bf2b6a8b39d0430433216bfc496e72103baaa5 +F test/savepoint.test ea1a6c08454e1a4edd973589bcf1fca83928ed09c48858fde1e8653b6daab278 F test/savepoint2.test 9b8543940572a2f01a18298c3135ad0c9f4f67d7 F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd @@ -1782,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 19c61ab79458936ff4dfca46cf4d1fb1ab16d7bdb5024f502eb4339ec4eef32c -R b8125f586e18293d9635394478d15d26 +P 1d8d4f689653ce80157740e339f7f1b42479bf90d82b176d8202d0a49f428398 +R ea40d6e9e89bd0f6f2c98f8376b01df3 U dan -Z 00401c2c33715960f0bf723cc059ce5c +Z 1339c3788ed308ea20667b613925c238 diff --git a/manifest.uuid b/manifest.uuid index ebdf4ef09b..defc4724b4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1d8d4f689653ce80157740e339f7f1b42479bf90d82b176d8202d0a49f428398 \ No newline at end of file +285d1c5904dd607457c6b5ff7be3d3191a90ca8b1fb0837327663c9910f978ac \ No newline at end of file diff --git a/test/savepoint.test b/test/savepoint.test index f196f8d2fc..16155547a3 100644 --- a/test/savepoint.test +++ b/test/savepoint.test @@ -806,7 +806,8 @@ do_test savepoint-11.6 { integrity_check savepoint-11.7 do_test savepoint-11.8 { execsql { ROLLBACK } - execsql { PRAGMA wal_checkpoint } + db close + sqlite3 db test.db file size test.db } {8192} From 7b812b6b16b7a4374857d104afbdcca938c13b87 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 5 Dec 2018 18:25:23 +0000 Subject: [PATCH 100/179] Add test cases to wal2concurrent.test. FossilOrigin-Name: ef9e7a87a4ce4fe9596aed2f880f2b9fc3526c00c48f2868c625a4c24be65abe --- manifest | 12 ++-- manifest.uuid | 2 +- test/wal2concurrent.test | 125 +++++++++++++++++++++++---------------- 3 files changed, 80 insertions(+), 59 deletions(-) diff --git a/manifest b/manifest index 13f57465e0..7506b17af3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stest\sscript\sproblem\son\sthis\sbranch. -D 2018-12-05T17:36:20.846 +C Add\stest\scases\sto\swal2concurrent.test. +D 2018-12-05T18:25:23.042 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -1606,7 +1606,7 @@ F test/vtab_err.test dcc8b7b9cb67522b3fe7a272c73856829dae4ab7fdb30399aea1b6981bd F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 F test/wal2.test a225bafac35a47765b890bacdeb57e5e81039f21cc18a1e8ce88eb76e56b843c -F test/wal2concurrent.test d0af64c3b113993553acfeeb64d207fb6d924e498fd953dbf31040a566ec10df +F test/wal2concurrent.test abb4c9d77e4cbd444aa95064f028805c2273c68440e45790b9da03f763fcad71 F test/wal2rewrite.test 6ca6f631ffcf871240beab5f02608913fd075c6d0d31310b026c8383c65c9f9c F test/wal2simple.test 8c9dfb8f1bca01a0deb57f7074cdb83865c2292e89b13f7a51a1c160dca3f5f4 F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2 @@ -1796,7 +1796,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bf309107dfc4d2abd68b9339c68aeaaffd278096bb2a5912e4a0b0c124fd6345 285d1c5904dd607457c6b5ff7be3d3191a90ca8b1fb0837327663c9910f978ac -R 95b9e871a13052b64e694448fd65a502 +P 692ddc280850ea2ff5d2fa3743f85fb123b4e4ebc452aae9c58b8d80d22b75b1 +R 70d5ec81123343dc4b5125cf054cd873 U dan -Z 1ca88420796234af6cf0aabbf996ef48 +Z e68147c0d113bb85826459be77b6127e diff --git a/manifest.uuid b/manifest.uuid index 30d584a429..5af0e23e2d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -692ddc280850ea2ff5d2fa3743f85fb123b4e4ebc452aae9c58b8d80d22b75b1 \ No newline at end of file +ef9e7a87a4ce4fe9596aed2f880f2b9fc3526c00c48f2868c625a4c24be65abe \ No newline at end of file diff --git a/test/wal2concurrent.test b/test/wal2concurrent.test index db0c8485b9..3375978bec 100644 --- a/test/wal2concurrent.test +++ b/test/wal2concurrent.test @@ -1,4 +1,4 @@ -# 2015 July 26 +# 2018 December 6 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: @@ -24,61 +24,82 @@ ifcapable !concurrent { #------------------------------------------------------------------------- # Warm-body test. # -sqlite3 db2 test.db -do_execsql_test 1.0 { - PRAGMA page_size = 1024; - CREATE TABLE t1(x); - CREATE TABLE t2(y); - PRAGMA journal_size_limit = 5000; - PRAGMA journal_mode = wal2; -} {5000 wal2} - -do_execsql_test 1.1 { - INSERT INTO t1 VALUES(1); - BEGIN CONCURRENT; - INSERT INTO t1 VALUES(2); -} {} - -do_test 1.2 { - execsql { +foreach tn {1 2} { + reset_db + sqlite3 db2 test.db + do_execsql_test 1.0 { + PRAGMA page_size = 1024; + CREATE TABLE t1(x); + CREATE TABLE t2(y); PRAGMA journal_size_limit = 5000; - INSERT INTO t1 VALUES(3) - } db2 - catchsql { COMMIT } -} {1 {database is locked}} - -do_catchsql_test 1.3 { COMMIT } {1 {database is locked}} -do_catchsql_test 1.4 { ROLLBACK } {0 {}} - -do_test 1.5 { - list [file size test.db-wal] [file size test.db-wal2] -} {2128 0} + PRAGMA journal_mode = wal2; + } {5000 wal2} + + do_execsql_test 1.1 { + INSERT INTO t1 VALUES(1); + BEGIN CONCURRENT; + INSERT INTO t1 VALUES(2); + } {} + + do_test 1.2 { + execsql { + PRAGMA journal_size_limit = 5000; + INSERT INTO t1 VALUES(3) + } db2 + catchsql { COMMIT } + } {1 {database is locked}} + + do_catchsql_test 1.3 { COMMIT } {1 {database is locked}} + do_catchsql_test 1.4 { ROLLBACK } {0 {}} + + do_test 1.5 { + list [file size test.db-wal] [file size test.db-wal2] + } {2128 0} + + do_execsql_test 1.6 { + BEGIN CONCURRENT; + INSERT INTO t1 VALUES(2); + } {} + + do_test 1.7 { + execsql { INSERT INTO t2 VALUES(randomblob(4000)) } db2 + list [file size test.db-wal] [file size test.db-wal2] + } {7368 0} + + if {$tn==1} { + do_test 1.8 { + execsql { + INSERT INTO t2 VALUES(1); + INSERT INTO t1 VALUES(5); + } db2 + list [file size test.db-wal] [file size test.db-wal2] + } {7368 2128} -do_execsql_test 1.6 { - BEGIN CONCURRENT; - INSERT INTO t1 VALUES(2); -} {} + do_catchsql_test 1.9 { COMMIT } {1 {database is locked}} + do_catchsql_test 1.10 { ROLLBACK } {0 {}} + db close + sqlite3 db test.db + do_execsql_test 1.11 { SELECT * FROM t1 } {1 3 5} + do_execsql_test 1.12 { SELECT count(*) FROM t2 } {2} + } else { + do_test 1.8 { + execsql { + INSERT INTO t2 VALUES(1); + } db2 + list [file size test.db-wal] [file size test.db-wal2] + } {7368 1080} -do_test 1.7 { - execsql { INSERT INTO t2 VALUES(randomblob(4000)) } db2 - list [file size test.db-wal] [file size test.db-wal2] -} {7368 0} + do_catchsql_test 1.9 { COMMIT } {0 {}} + db close + sqlite3 db test.db + do_execsql_test 1.11 { SELECT * FROM t1 } {1 3 2} + do_execsql_test 1.12 { SELECT count(*) FROM t2 } {2} -do_test 1.8 { - execsql { - INSERT INTO t2 VALUES(1); - INSERT INTO t1 VALUES(5); - } db2 - list [file size test.db-wal] [file size test.db-wal2] -} {7368 2128} - -do_catchsql_test 1.9 { COMMIT } {1 {database is locked}} -do_catchsql_test 1.10 { ROLLBACK } {0 {}} - -db close -sqlite3 db test.db -do_execsql_test 1.11 { SELECT * FROM t1 } {1 3 5} -do_execsql_test 1.12 { SELECT count(*) FROM t2 } {2} + do_test 1.13 { + list [file size test.db-wal] [file size test.db-wal2] + } {7368 2128} + } +} finish_test From d597b966d9ffe95daa6a7143091d87916c6225f1 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 6 Dec 2018 16:54:44 +0000 Subject: [PATCH 101/179] Add test file wal2snapshot.test that should have been added as part of an earlier commit. FossilOrigin-Name: f6baa7e1163ed5f61375b0554337030fec23e8a9f60c6412e1b5d626961e93f9 --- manifest | 11 +++--- manifest.uuid | 2 +- test/wal2snapshot.test | 84 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 6 deletions(-) create mode 100644 test/wal2snapshot.test diff --git a/manifest b/manifest index 5b4bda667c..07d0505ca6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stest\sscript\sproblem\son\sthis\sbranch. -D 2018-12-05T17:31:24.866 +C Add\stest\sfile\swal2snapshot.test\sthat\sshould\shave\sbeen\sadded\sas\spart\sof\san\nearlier\scommit. +D 2018-12-06T16:54:44.030 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -1595,6 +1595,7 @@ F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 F test/wal2.test 155b9efa999bdb38ce1cd729b9a4fcdbffd6b88be27f039bad1d2929d287d918 F test/wal2rewrite.test 6ca6f631ffcf871240beab5f02608913fd075c6d0d31310b026c8383c65c9f9c F test/wal2simple.test 8c9dfb8f1bca01a0deb57f7074cdb83865c2292e89b13f7a51a1c160dca3f5f4 +F test/wal2snapshot.test 95a919e1c73dee0e0212d10931d03cc1116f68a0ff603163e551aaa5ac7025c1 F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9 @@ -1782,7 +1783,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1d8d4f689653ce80157740e339f7f1b42479bf90d82b176d8202d0a49f428398 -R ea40d6e9e89bd0f6f2c98f8376b01df3 +P 285d1c5904dd607457c6b5ff7be3d3191a90ca8b1fb0837327663c9910f978ac +R c6176f33f0706e46f32e37a55f5b7c74 U dan -Z 1339c3788ed308ea20667b613925c238 +Z dba2fba0086f2ef35808e528fd424f75 diff --git a/manifest.uuid b/manifest.uuid index defc4724b4..60435bf318 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -285d1c5904dd607457c6b5ff7be3d3191a90ca8b1fb0837327663c9910f978ac \ No newline at end of file +f6baa7e1163ed5f61375b0554337030fec23e8a9f60c6412e1b5d626961e93f9 \ No newline at end of file diff --git a/test/wal2snapshot.test b/test/wal2snapshot.test new file mode 100644 index 0000000000..9f461db7d6 --- /dev/null +++ b/test/wal2snapshot.test @@ -0,0 +1,84 @@ +# 2018 December 5 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL2" mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +set testprefix wal2snapshot +ifcapable !wal {finish_test ; return } +ifcapable !snapshot {finish_test; return} + +foreach {tn mode} {1 wal 2 wal2} { + reset_db + do_execsql_test $tn.1 "PRAGMA journal_mode = $mode" $mode + + do_execsql_test $tn.2 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + BEGIN; + } + + # Check that sqlite3_snapshot_get() is an error for a wal2 db. + # + if {$tn==1} { + do_test 1.3 { + set S [sqlite3_snapshot_get db main] + sqlite3_snapshot_free $S + } {} + } else { + do_test 2.3 { + list [catch { sqlite3_snapshot_get db main } msg] $msg + } {1 SQLITE_ERROR} + } + + # Check that sqlite3_snapshot_recover() is an error for a wal2 db. + # + do_execsql_test $tn.4 COMMIT + if {$tn==1} { + do_test 1.5 { + sqlite3_snapshot_recover db main + } {} + } else { + do_test 2.5 { + list [catch { sqlite3_snapshot_recover db main } msg] $msg + } {1 SQLITE_ERROR} + } + + # Check that sqlite3_snapshot_open() is an error for a wal2 db. + # + if {$tn==1} { + do_test 1.6 { + execsql BEGIN + set SNAPSHOT [sqlite3_snapshot_get_blob db main] + sqlite3_snapshot_open_blob db main $SNAPSHOT + execsql COMMIT + } {} + } else { + do_test 2.6 { + execsql BEGIN + set res [ + list [catch { sqlite3_snapshot_open_blob db main $SNAPSHOT } msg] $msg + ] + execsql COMMIT + set res + } {1 SQLITE_ERROR} + } +} + + +finish_test + + From 24fb9ffbc137b67829f240431b4ea58890bc437e Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 6 Dec 2018 18:58:58 +0000 Subject: [PATCH 102/179] Experiment with using an atomic CPU primitive instead of a mutex for intra-process locking with the unix-excl VFS. FossilOrigin-Name: 8f4cb9dd3396bcaaf85dcdb4e3ae3c96f28a4d71d72665c4abf7c221370be626 --- manifest | 26 ++++++---- manifest.uuid | 2 +- src/os_unix.c | 85 +++++++++++++++++++++++++++++++ src/test_vfs.c | 54 ++++++++++++++++++++ test/lock_common.tcl | 10 ++-- test/permutations.test | 10 ++++ test/shmlock.test | 111 +++++++++++++++++++++++++++++++++++++++++ test/wal.test | 90 +++++++++++++++++---------------- 8 files changed, 328 insertions(+), 60 deletions(-) create mode 100644 test/shmlock.test diff --git a/manifest b/manifest index 720ed6ca6b..f8b2c4a247 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Invoking\sthe\ssqlite3_trace()\sor\ssqlite3_trace_v2()\sinterfaces\scancels\nany\ssqlite3_profile()\sthat\sis\srunning. -D 2018-12-06T03:59:25.918 +C Experiment\swith\susing\san\satomic\sCPU\sprimitive\sinstead\sof\sa\smutex\sfor\nintra-process\slocking\swith\sthe\sunix-excl\sVFS. +D 2018-12-06T18:58:58.578 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -491,7 +491,7 @@ F src/os.c 8aeb0b0f40f8f5b0da03fe49706695adaf42d2f516ab95abc72e86c245e119de F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c f6e91b8fd82af7afbfd073c4974ad6cdb8e62d9f65ceddb45167835a0567fdc0 +F src/os_unix.c dcf08ca4f7092507015391b078f9ab866e8fcc6fba22560086fe1eec031edd18 F src/os_win.c 85d9e532d0444ab6c16d7431490c2e279e282aa0917b0e988996b1ae0de5c5a0 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 75e0f3cfa3962c714f519f8a3d1e67ecca1c91de0e010a036b988e40ce9e4c73 @@ -563,7 +563,7 @@ F src/test_syscall.c 1073306ba2e9bfc886771871a13d3de281ed3939 F src/test_tclsh.c 06317648b0d85a85fd823f7973b55535c59a3156c1ef59394fe511f932cfa78d F src/test_tclvar.c 33ff42149494a39c5fbb0df3d25d6fafb2f668888e41c0688d07273dcb268dfc F src/test_thread.c 911d15fb14e19c0c542bdc8aabf981c2f10a4858 -F src/test_vfs.c 112f1f9271c33c211812e0e681830a84262dac065da58579ff49f9cefec97d4f +F src/test_vfs.c 194b8e78e9b4279bc69693cb5cfc343e0a08482c430a571ec96d80440e337a44 F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_windirent.c a895e2c068a06644eef91a7f0a32182445a893b9a0f33d0cdb4283dca2486ac1 F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a90484215 @@ -1078,7 +1078,7 @@ F test/lock4.test 27143363eda1622f03c133efc8db808fc331afd973486cb571ea71cd717d37 F test/lock5.test c6c5e0ebcb21c61a572870cc86c0cb9f14cede38 F test/lock6.test ad5b387a3a8096afd3c68a55b9535056431b0cf5 F test/lock7.test 49f1eaff1cdc491cc5dee3669f3c671d9f172431 -F test/lock_common.tcl 7ffb45accf6ee91c736df9bafe0806a44358f035 +F test/lock_common.tcl 2f3f7f2e9637f93ccf609df48ef5b27a50278b6b1cd752b445d52262e5841413 F test/lookaside.test 5a828e7256f1ee4da8e1bdaa03373a3ccdb0f1ff98dfa82e9b76cb41a45b1083 F test/main.test 6bbb3999fd461eb8fb335cbab97409a3d7f91bbb8da60635e8be3e4a04a77772 F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9 @@ -1181,7 +1181,7 @@ F test/parser1.test 6ccdf5e459a5dc4673d3273dc311a7e9742ca952dd0551a6a6320d27035c F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test cf0b4e498db1d0143c19641d4420df7cc27fab2c95ed0abd2c7c5753beab25b8 +F test/permutations.test c2e5f618a71d1a7e50216325dd056f6ec16439b3c98d87bcd2df381297e5dc97 F test/pg_common.tcl 301ac19c1a52fd55166d26db929b3b89165c634d52b5f8ad76ea8cb06960db30 F test/pragma.test c267bf02742c823a191960895b3d52933cebd7beee26757d1ed694f213fcd867 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f @@ -1285,6 +1285,7 @@ F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35 F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f F test/shell8.test 96be02ea0c21f05b24c1883d7b711a1fa8525a68ab7b636aacf6057876941013 +F test/shmlock.test 2d8d2401fb001414a817d61b2c4983215d546c71d48da517a0a5015422997628 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5 F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce @@ -1592,7 +1593,7 @@ F test/vtabJ.test d7b73675708cf63cfcb9d443bb451fc01a028347275b7311e51f9fdf3ca675 F test/vtab_alter.test 736e66fb5ec7b4fee58229aa3ada2f27ec58bc58c00edae4836890c3784c6783 F test/vtab_err.test dcc8b7b9cb67522b3fe7a272c73856829dae4ab7fdb30399aea1b6981bda2b65 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad -F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 +F test/wal.test cdf0ca6cc0447520d19ef1c83287824ebeb3e82d75af856511ba96841a79fc9b F test/wal2.test 155b9efa999bdb38ce1cd729b9a4fcdbffd6b88be27f039bad1d2929d287d918 F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c @@ -1782,7 +1783,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3a2c047989facc3461c63a2f9eed412014c951035a80da47c52a70139fb552de -R e33573679d2b83476b49b39d100df865 -U drh -Z 9b63335dca170db3a75ac781c84ad8b2 +P ec63d3506bd429560077f82a4c5ed9d189780789fe1c134fff4f3b8733be1a3f +R 421ea33fec1ec5096b7fb80f863def19 +T *branch * mutexfree-shmlock +T *sym-mutexfree-shmlock * +T -sym-trunk * +U dan +Z 737072d87778273578811397782f507c diff --git a/manifest.uuid b/manifest.uuid index 787485caf5..f3e776b8e7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec63d3506bd429560077f82a4c5ed9d189780789fe1c134fff4f3b8733be1a3f \ No newline at end of file +8f4cb9dd3396bcaaf85dcdb4e3ae3c96f28a4d71d72665c4abf7c221370be626 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index f20763e5b0..946c5d3d65 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -46,6 +46,9 @@ #include "sqliteInt.h" #if SQLITE_OS_UNIX /* This file is used on unix only */ +/* Turn this feature on in all builds for now */ +#define SQLITE_MUTEXFREE_SHMLOCK 1 + /* ** There are various methods for file locking used for concurrency ** control: @@ -4239,8 +4242,38 @@ struct unixShmNode { u8 sharedMask; /* Mask of shared locks held */ u8 nextShmId; /* Next available unixShm.id value */ #endif + +#ifdef SQLITE_MUTEXFREE_SHMLOCK + /* In unix-excl mode, if SQLITE_MUTEXFREE_SHMLOCK is defined, all locks + ** are stored in the following 64-bit value. There are in total 8 + ** shm-locking slots, each of which are assigned 8-bits from the 64-bit + ** value. The least-significant 8 bits correspond to shm-locking slot + ** 0, and so on. + ** + ** If the 8-bits corresponding to a shm-locking locking slot are set to + ** 0xFF, then a write-lock is held on the slot. Or, if they are set to + ** a non-zero value smaller than 0xFF, then they represent the total + ** number of read-locks held on the slot. There is no way to distinguish + ** between a write-lock and 255 read-locks. */ + u64 lockmask; +#endif }; +/* +** Atomic CAS primitive used in multi-process mode. Equivalent to: +** +** int unixCompareAndSwap(u32 *ptr, u32 oldval, u32 newval){ +** if( *ptr==oldval ){ +** *ptr = newval; +** return 1; +** } +** return 0; +** } +*/ +#define unixCompareAndSwap(ptr,oldval,newval) \ + __sync_bool_compare_and_swap(ptr,oldval,newval) + + /* ** Structure used internally by this VFS to record the state of an ** open shared memory connection. @@ -4800,6 +4833,58 @@ static int unixShmLock( mask = (1<<(ofst+n)) - (1<1 || mask==(1<pInode->bProcessLock ){ + + while( 1 ){ + u64 lockmask = pShmNode->lockmask; + u64 newmask = lockmask; + int i; + for(i=ofst; i> (ix8)) & 0xFF; + if( flags & SQLITE_SHM_UNLOCK ){ + if( flags & SQLITE_SHM_EXCLUSIVE ){ + if( p->exclMask & (1 << i) ){ + newmask = newmask & ~((u64)0xFF<sharedMask & (1 << i) ){ + newmask = newmask & ~((u64)0xFF<exclMask & (1 << i))==0 ){ + newmask = newmask | ((u64)0xFF<sharedMask & (1 << i))==0 ){ + newmask = newmask & ~((u64)0xFF<lockmask, lockmask, newmask) ) break; + } + + if( flags & SQLITE_SHM_UNLOCK ){ + p->sharedMask &= ~mask; + p->exclMask &= ~mask; + }else if( flags & SQLITE_SHM_EXCLUSIVE ){ + p->exclMask |= mask; + }else{ + p->sharedMask |= mask; + } + + return SQLITE_OK; + } +#endif + sqlite3_mutex_enter(pShmNode->pShmMutex); if( flags & SQLITE_SHM_UNLOCK ){ u16 allMask = 0; /* Mask of locks held by siblings */ diff --git a/src/test_vfs.c b/src/test_vfs.c index 4a98ac214c..645b1447c2 100644 --- a/src/test_vfs.c +++ b/src/test_vfs.c @@ -1563,8 +1563,62 @@ static int SQLITE_TCLAPI testvfs_cmd( return TCL_ERROR; } +extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); +extern const char *sqlite3ErrName(int); + +/* +** tclcmd: vfs_shmlock DB DBNAME (shared|exclusive) (lock|unlock) OFFSET N +*/ +static int SQLITE_TCLAPI test_vfs_shmlock( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + const char *azArg1[] = {"shared", "exclusive", 0}; + const char *azArg2[] = {"lock", "unlock", 0}; + sqlite3 *db = 0; + int rc = SQLITE_OK; + const char *zDbname = 0; + int iArg1 = 0; + int iArg2 = 0; + int iOffset = 0; + int n = 0; + sqlite3_file *pFd; + + if( objc!=7 ){ + Tcl_WrongNumArgs(interp, 1, objv, + "DB DBNAME (shared|exclusive) (lock|unlock) OFFSET N" + ); + return TCL_ERROR; + } + + zDbname = Tcl_GetString(objv[2]); + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) + || Tcl_GetIndexFromObj(interp, objv[3], azArg1, "ARG", 0, &iArg1) + || Tcl_GetIndexFromObj(interp, objv[4], azArg2, "ARG", 0, &iArg2) + || Tcl_GetIntFromObj(interp, objv[5], &iOffset) + || Tcl_GetIntFromObj(interp, objv[6], &n) + ){ + return TCL_ERROR; + } + + sqlite3_file_control(db, zDbname, SQLITE_FCNTL_FILE_POINTER, (void*)&pFd); + if( pFd==0 ){ + return TCL_ERROR; + } + rc = pFd->pMethods->xShmLock(pFd, iOffset, n, + (iArg1==0 ? SQLITE_SHM_SHARED : SQLITE_SHM_EXCLUSIVE) + | (iArg2==0 ? SQLITE_SHM_LOCK : SQLITE_SHM_UNLOCK) + ); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + return TCL_OK; +} + + int Sqlitetestvfs_Init(Tcl_Interp *interp){ Tcl_CreateObjCommand(interp, "testvfs", testvfs_cmd, 0, 0); + Tcl_CreateObjCommand(interp, "vfs_shmlock", test_vfs_shmlock, 0, 0); return TCL_OK; } diff --git a/test/lock_common.tcl b/test/lock_common.tcl index a758e7af2e..3e1821bab0 100644 --- a/test/lock_common.tcl +++ b/test/lock_common.tcl @@ -15,18 +15,20 @@ proc do_multiclient_test {varname script} { - foreach code [list { + foreach {tn code} [list 1 { if {[info exists ::G(valgrind)]} { db close ; continue } set ::code2_chan [launch_testfixture] set ::code3_chan [launch_testfixture] proc code2 {tcl} { testfixture $::code2_chan $tcl } proc code3 {tcl} { testfixture $::code3_chan $tcl } - set tn 1 - } { + } 2 { proc code2 {tcl} { uplevel #0 $tcl } proc code3 {tcl} { uplevel #0 $tcl } - set tn 2 }] { + # Do not run multi-process tests with the unix-excl VFS. + # + if {$tn==1 && [permutation]=="unix-excl"} continue + faultsim_delete_and_reopen proc code1 {tcl} { uplevel #0 $tcl } diff --git a/test/permutations.test b/test/permutations.test index 6aa812e8af..a4c21e67d8 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -607,6 +607,16 @@ test_suite "onefile" -description { rollback.test select1.test select2.test select3.test } +# Run some tests using the "unix-excl" VFS. +# +test_suite "unix-excl" -description { + Run some tests using the "unix-excl" VFS +} -initialize { + set ::G(perm:sqlite3_args) [list -vfs unix-excl] +} -files { + shmlock.test +} + # Run some tests using UTF-16 databases. # test_suite "utf16" -description { diff --git a/test/shmlock.test b/test/shmlock.test new file mode 100644 index 0000000000..ac79ac44d3 --- /dev/null +++ b/test/shmlock.test @@ -0,0 +1,111 @@ +# 2018 December 6 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +set testprefix shmlock + +ifcapable !wal {finish_test ; return } + +sqlite3 db2 test.db +sqlite3 db3 test.db + +do_execsql_test 1.0 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); +} {wal} +do_test 1.1 { execsql { SELECT * FROM t1 } db2 } {1 2} +do_test 1.2 { execsql { SELECT * FROM t1 } db3 } {1 2} + +foreach {tn dbhandle cmd res} { + 1 db {shared lock 7 1} OK + 2 db2 {exclusive lock 7 1} BUSY + 3 db {shared unlock 7 1} OK + 4 db2 {exclusive lock 7 1} OK + 5 db {shared lock 7 1} BUSY + 6 db {exclusive lock 7 1} BUSY + 7 db2 {exclusive unlock 7 1} OK + + 8 db {exclusive lock 0 8} OK + 9 db {exclusive unlock 0 8} OK + 10 db2 {exclusive lock 0 8} OK + 11 db2 {exclusive unlock 0 8} OK + + 12 db {shared lock 0 1} OK + 13 db2 {shared lock 0 1} OK + 14 db3 {shared lock 0 1} OK + 15 db3 {shared unlock 0 1} OK + 16 db3 {exclusive lock 0 1} BUSY + 17 db2 {shared unlock 0 1} OK + 18 db3 {exclusive lock 0 1} BUSY + 19 db {shared unlock 0 1} OK + 20 db3 {exclusive lock 0 1} OK + 21 db3 {exclusive unlock 0 1} OK + + 22 db {shared lock 3 1} OK + 23 db2 {exclusive lock 2 2} BUSY + 24 db {shared lock 2 1} OK + 25 db2 {exclusive lock 0 5} BUSY + 26 db2 {exclusive lock 0 4} BUSY + 27 db2 {exclusive lock 0 3} BUSY + 28 db {shared unlock 3 1} OK + 29 db2 {exclusive lock 2 2} BUSY + 28 db {shared unlock 2 1} OK + 29 db2 {exclusive lock 2 2} OK + 29 db2 {exclusive unlock 2 2} OK +} { + do_test 1.3.$tn [list vfs_shmlock $dbhandle main {*}$cmd] "SQLITE_$res" +} + +db close +db2 close +db3 close + +if {[permutation]=="unix-excl"} { + do_test 2.0 { + for {set i 0} {$i < 256} {incr i} { + sqlite3 db$i test.db + execsql { SELECT * FROM t1 } db$i + } + for {set i 0} {$i < 255} {incr i} { + set rc [vfs_shmlock db$i main shared lock 4 1] + if {$rc != "SQLITE_OK"} { error $rc } + } + + vfs_shmlock db255 main shared lock 4 1 + } {SQLITE_BUSY} + + do_test 2.1 { vfs_shmlock db255 main exclusive lock 4 1 } SQLITE_BUSY + do_test 2.2 { vfs_shmlock db0 main shared unlock 4 1 } SQLITE_OK + do_test 2.3 { vfs_shmlock db255 main shared lock 4 1 } SQLITE_OK + do_test 2.4 { vfs_shmlock db255 main shared unlock 4 1 } SQLITE_OK + do_test 2.5 { vfs_shmlock db255 main exclusive lock 4 1 } SQLITE_BUSY + + do_test 2.6 { + for {set i 1} {$i < 255} {incr i} { + set rc [vfs_shmlock db255 main exclusive lock 4 1] + if {$rc != "SQLITE_BUSY"} { error $rc } + set rc [vfs_shmlock db$i main shared unlock 4 1] + if {$rc != "SQLITE_OK"} { error $rc } + } + + vfs_shmlock db255 main exclusive lock 4 1 + } {SQLITE_OK} + + vfs_shmlock db255 main exclusive unlock 4 1 +} + +finish_test + + diff --git a/test/wal.test b/test/wal.test index bb164bb76a..a003b6ad20 100644 --- a/test/wal.test +++ b/test/wal.test @@ -1297,51 +1297,53 @@ do_test wal-19.4 { # At one point, SQLite was failing to grow the mapping of the wal-index # file in step 3 and the checkpoint was corrupting the database file. # -do_test wal-20.1 { - catch {db close} - forcedelete test.db test.db-wal test.db-journal - sqlite3 db test.db - execsql { - PRAGMA journal_mode = WAL; - CREATE TABLE t1(x); - INSERT INTO t1 VALUES(randomblob(900)); - SELECT count(*) FROM t1; - } -} {wal 1} -do_test wal-20.2 { - set ::buddy [launch_testfixture] - testfixture $::buddy { +if {[permutation]!="unix-excl"} { + do_test wal-20.1 { + catch {db close} + forcedelete test.db test.db-wal test.db-journal sqlite3 db test.db - db transaction { db eval { - PRAGMA wal_autocheckpoint = 0; - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 2 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 32 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 64 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 128 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 256 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 512 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 1024 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 2048 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4096 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8192 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16384 */ - } } - } -} {0} -do_test wal-20.3 { - close $::buddy - execsql { PRAGMA wal_checkpoint } - execsql { SELECT count(*) FROM t1 } -} {16384} -do_test wal-20.4 { - db close - sqlite3 db test.db - execsql { SELECT count(*) FROM t1 } -} {16384} -integrity_check wal-20.5 + execsql { + PRAGMA journal_mode = WAL; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(randomblob(900)); + SELECT count(*) FROM t1; + } + } {wal 1} + do_test wal-20.2 { + set ::buddy [launch_testfixture] + testfixture $::buddy { + sqlite3 db test.db + db transaction { db eval { + PRAGMA wal_autocheckpoint = 0; + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 2 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 32 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 64 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 128 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 256 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 512 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 1024 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 2048 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4096 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8192 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16384 */ + } } + } + } {0} + do_test wal-20.3 { + close $::buddy + execsql { PRAGMA wal_checkpoint } + execsql { SELECT count(*) FROM t1 } + } {16384} + do_test wal-20.4 { + db close + sqlite3 db test.db + execsql { SELECT count(*) FROM t1 } + } {16384} + integrity_check wal-20.5 +} catch { db2 close } catch { db close } From b0b27ab5c65b49db409379a09852b3073156622f Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 7 Dec 2018 20:25:14 +0000 Subject: [PATCH 103/179] Add multi-threaded performance test program "tserver" to this branch. Fix bugs in the begin-concurrent/wal2 integration revealed by the same. FossilOrigin-Name: 7bd3b35661d7d0e51113b9e4b15a0ab7f8e26edeafb43941ef5b44bb94df5109 --- manifest | 14 +- manifest.uuid | 2 +- src/wal.c | 16 +- tool/tserver.c | 614 ++++++++++++++++++++++++++++++++++++++++++ tool/tserver_test.tcl | 283 +++++++++++++++++++ 5 files changed, 918 insertions(+), 11 deletions(-) create mode 100644 tool/tserver.c create mode 100644 tool/tserver_test.tcl diff --git a/manifest b/manifest index 7506b17af3..32739465ba 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\scases\sto\swal2concurrent.test. -D 2018-12-05T18:25:23.042 +C Add\smulti-threaded\sperformance\stest\sprogram\s"tserver"\sto\sthis\sbranch.\sFix\sbugs\nin\sthe\sbegin-concurrent/wal2\sintegration\srevealed\sby\sthe\ssame. +D 2018-12-07T20:25:14.317 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -593,7 +593,7 @@ F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392 F src/vtab.c 70188a745dc4e57d26e942681ff4b2912b7c8249ad5de3f60f0677b4337bcfaa F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 43a7e48cea522cbb8eaf198b5f47ec4aea3bf89ae41c65a5fa56d77c6b2125e0 +F src/wal.c 1a564ef45c6443c52d1c576d0b7b9fb969a91457ed2cf286d98d900e88a5f73f F src/wal.h c398e0269e8f37495cedb63b5e288c2aac6f6d103d05fb55f4affec21311615d F src/walker.c fb94aadc9099ff9c6506d0a8b88d51266005bcaa265403f3d7caf732a562eb66 F src/where.c 3818e8a736a05d2cb194e64399af707e367fbcc5c251d785804d02eaf121288e @@ -1769,6 +1769,8 @@ F tool/srcck1.c 371de5363b70154012955544f86fdee8f6e5326f F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d F tool/symbols.sh c5a617b8c61a0926747a56c65f5671ef8ac0e148 +F tool/tserver.c 46ef565f79b326a86b696ae255548cf841fafae10dda7b6a2027b323f4ee6dac +F tool/tserver_test.tcl 777e1e4f26c1bf47fecdb61ce46e1db36c209a81956ccfd9e2d957377b54974f F tool/varint.c 5d94cb5003db9dbbcbcc5df08d66f16071aee003 F tool/vdbe-compress.tcl 5926c71f9c12d2ab73ef35c29376e756eb68361c F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f @@ -1796,7 +1798,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 692ddc280850ea2ff5d2fa3743f85fb123b4e4ebc452aae9c58b8d80d22b75b1 -R 70d5ec81123343dc4b5125cf054cd873 +P ef9e7a87a4ce4fe9596aed2f880f2b9fc3526c00c48f2868c625a4c24be65abe +R d4106d386120e66ef9e2fbd9ae6bf9bf U dan -Z e68147c0d113bb85826459be77b6127e +Z 4d7f96aaf6dd169d1c2711123641179c diff --git a/manifest.uuid b/manifest.uuid index 5af0e23e2d..ca1125993a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ef9e7a87a4ce4fe9596aed2f880f2b9fc3526c00c48f2868c625a4c24be65abe \ No newline at end of file +7bd3b35661d7d0e51113b9e4b15a0ab7f8e26edeafb43941ef5b44bb94df5109 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 244ff5c57c..086334bbd5 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2435,7 +2435,10 @@ static int walCheckpoint( } /* Release the reader lock held while backfilling */ - walUnlockExclusive(pWal, WAL_READ_LOCK(bWal2 ? 1 + iCkpt*2 : 0), 1); + if( bWal2 ){ + walUnlockExclusive(pWal, WAL_READ_LOCK(1 + iCkpt*2), 1); + } + walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1); } if( rc==SQLITE_BUSY ){ @@ -3861,11 +3864,16 @@ int sqlite3WalLockForCommit( u8 aNew[4]; u8 *aOld = &((u8*)pPage1->pData)[40]; int sz; - i64 iOffset; + i64 iOff; + int iFrame = sLoc.iZero + i; + int iWal = 0; + if( bWal2 ){ + iWal = walExternalDecode(iFrame, &iFrame); + } sz = pWal->hdr.szPage; sz = (sz&0xfe00) + ((sz&0x0001)<<16); - iOffset = walFrameOffset(i+sLoc.iZero, sz)+WAL_FRAME_HDRSIZE+40; - rc = sqlite3OsRead(pWal->apWalFd[0], aNew,sizeof(aNew),iOffset); + iOff = walFrameOffset(iFrame, sz) + WAL_FRAME_HDRSIZE + 40; + rc = sqlite3OsRead(pWal->apWalFd[iWal],aNew,sizeof(aNew),iOff); if( rc==SQLITE_OK && memcmp(aOld, aNew, sizeof(aNew)) ){ rc = SQLITE_BUSY_SNAPSHOT; } diff --git a/tool/tserver.c b/tool/tserver.c new file mode 100644 index 0000000000..c2dc35b4f8 --- /dev/null +++ b/tool/tserver.c @@ -0,0 +1,614 @@ +/* +** 2017 June 7 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** Simple multi-threaded server used for informal testing of concurrency +** between connections in different threads. Listens for tcp/ip connections +** on port 9999 of the 127.0.0.1 interface only. To build: +** +** gcc -g $(TOP)/tool/tserver.c sqlite3.o -lpthread -o tserver +** +** To run using "x.db" as the db file: +** +** ./tserver x.db +** +** To connect, open a client socket on port 9999 and start sending commands. +** Commands are either SQL - which must be terminated by a semi-colon, or +** dot-commands, which must be terminated by a newline. If an SQL statement +** is seen, it is prepared and added to an internal list. +** +** Dot-commands are: +** +** .list Display all SQL statements in the list. +** .quit Disconnect. +** .run Run all SQL statements in the list. +** .repeats N Configure the number of repeats per ".run". +** .seconds N Configure the number of seconds to ".run" for. +** .mutex_commit Add a "COMMIT" protected by a g.commit_mutex +** to the current SQL. +** .stop Stop the tserver process - exit(0). +** +** Example input: +** +** BEGIN; +** INSERT INTO t1 VALUES(randomblob(10), randomblob(100)); +** INSERT INTO t1 VALUES(randomblob(10), randomblob(100)); +** INSERT INTO t1 VALUES(randomblob(10), randomblob(100)); +** COMMIT; +** .repeats 100000 +** .run +** +*/ +#define TSERVER_PORTNUMBER 9999 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sqlite3.h" + +#define TSERVER_DEFAULT_CHECKPOINT_THRESHOLD 3900 + +/* Global variables */ +struct TserverGlobal { + char *zDatabaseName; /* Database used by this server */ + char *zVfs; + sqlite3_mutex *commit_mutex; + sqlite3 *db; /* Global db handle */ + + /* The following use native pthreads instead of a portable interface. This + ** is because a condition variable, as well as a mutex, is required. */ + pthread_mutex_t ckpt_mutex; + pthread_cond_t ckpt_cond; + int nThreshold; /* Checkpoint when wal is this large */ + int bCkptRequired; /* True if wal checkpoint is required */ + int nRun; /* Number of clients in ".run" */ + int nWait; /* Number of clients waiting on ckpt_cond */ +}; + +static struct TserverGlobal g = {0}; + +typedef struct ClientSql ClientSql; +struct ClientSql { + sqlite3_stmt *pStmt; + int bMutex; +}; + +typedef struct ClientCtx ClientCtx; +struct ClientCtx { + sqlite3 *db; /* Database handle for this client */ + int fd; /* Client fd */ + int nRepeat; /* Number of times to repeat SQL */ + int nSecond; /* Number of seconds to run for */ + ClientSql *aPrepare; /* Array of prepared statements */ + int nPrepare; /* Valid size of apPrepare[] */ + int nAlloc; /* Allocated size of apPrepare[] */ + + int nClientThreshold; /* Threshold for checkpointing */ + int bClientCkptRequired; /* True to do a checkpoint */ +}; + +static int is_eol(int i){ + return (i=='\n' || i=='\r'); +} +static int is_whitespace(int i){ + return (i==' ' || i=='\t' || is_eol(i)); +} + +/* +** Implementation of SQL scalar function usleep(). +*/ +static void usleepFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int nUs; + sqlite3_vfs *pVfs = (sqlite3_vfs*)sqlite3_user_data(context); + assert( argc==1 ); + nUs = sqlite3_value_int64(argv[0]); + pVfs->xSleep(pVfs, nUs); +} + +static void trim_string(const char **pzStr, int *pnStr){ + const char *zStr = *pzStr; + int nStr = *pnStr; + + while( nStr>0 && is_whitespace(zStr[0]) ){ + zStr++; + nStr--; + } + while( nStr>0 && is_whitespace(zStr[nStr-1]) ){ + nStr--; + } + + *pzStr = zStr; + *pnStr = nStr; +} + +static int send_message(ClientCtx *p, const char *zFmt, ...){ + char *zMsg; + va_list ap; /* Vararg list */ + va_start(ap, zFmt); + int res = -1; + + zMsg = sqlite3_vmprintf(zFmt, ap); + if( zMsg ){ + res = write(p->fd, zMsg, strlen(zMsg)); + } + sqlite3_free(zMsg); + va_end(ap); + + return (res<0); +} + +static int handle_some_sql(ClientCtx *p, const char *zSql, int nSql){ + const char *zTail = zSql; + int nTail = nSql; + int rc = SQLITE_OK; + + while( rc==SQLITE_OK ){ + if( p->nPrepare>=p->nAlloc ){ + int nByte = (p->nPrepare+32) * sizeof(ClientSql); + ClientSql *aNew = sqlite3_realloc(p->aPrepare, nByte); + if( aNew ){ + memset(&aNew[p->nPrepare], 0, sizeof(ClientSql)*32); + p->aPrepare = aNew; + p->nAlloc = p->nPrepare+32; + }else{ + rc = SQLITE_NOMEM; + break; + } + } + rc = sqlite3_prepare_v2( + p->db, zTail, nTail, &p->aPrepare[p->nPrepare].pStmt, &zTail + ); + if( rc!=SQLITE_OK ){ + send_message(p, "error - %s (eec=%d)\n", sqlite3_errmsg(p->db), + sqlite3_extended_errcode(p->db) + ); + rc = 1; + break; + } + if( p->aPrepare[p->nPrepare].pStmt==0 ){ + break; + } + p->nPrepare++; + nTail = nSql - (zTail-zSql); + rc = send_message(p, "ok (%d SQL statements)\n", p->nPrepare); + } + + return rc; +} + +static sqlite3_int64 get_timer(void){ + struct timeval t; + gettimeofday(&t, 0); + return ((sqlite3_int64)t.tv_usec / 1000) + ((sqlite3_int64)t.tv_sec * 1000); +} + +static void clear_sql(ClientCtx *p){ + int j; + for(j=0; jnPrepare; j++){ + sqlite3_finalize(p->aPrepare[j].pStmt); + } + p->nPrepare = 0; +} + +/* +** The sqlite3_wal_hook() callback used by all client database connections. +*/ +static int clientWalHook(void *pArg, sqlite3 *db, const char *zDb, int nFrame){ + if( g.nThreshold>0 ){ + if( nFrame>=g.nThreshold ){ + g.bCkptRequired = 1; + } + }else{ + ClientCtx *pCtx = (ClientCtx*)pArg; + if( pCtx->nClientThreshold && nFrame>=pCtx->nClientThreshold ){ + pCtx->bClientCkptRequired = 1; + } + } + return SQLITE_OK; +} + +static int handle_run_command(ClientCtx *p){ + int i, j; + int nBusy = 0; + sqlite3_int64 t0 = get_timer(); + sqlite3_int64 t1 = t0; + int nT1 = 0; + int nTBusy1 = 0; + int rc = SQLITE_OK; + + pthread_mutex_lock(&g.ckpt_mutex); + g.nRun++; + pthread_mutex_unlock(&g.ckpt_mutex); + + + for(j=0; (p->nRepeat<=0 || jnRepeat) && rc==SQLITE_OK; j++){ + sqlite3_int64 t2; + + for(i=0; inPrepare && rc==SQLITE_OK; i++){ + sqlite3_stmt *pStmt = p->aPrepare[i].pStmt; + + /* If the bMutex flag is set, grab g.commit_mutex before executing + ** the SQL statement (which is always "COMMIT" in this case). */ + if( p->aPrepare[i].bMutex ){ + sqlite3_mutex_enter(g.commit_mutex); + } + + /* Execute the statement */ + while( sqlite3_step(pStmt)==SQLITE_ROW ); + rc = sqlite3_reset(pStmt); + + /* Relinquish the g.commit_mutex mutex if required. */ + if( p->aPrepare[i].bMutex ){ + sqlite3_mutex_leave(g.commit_mutex); + } + + if( (rc & 0xFF)==SQLITE_BUSY ){ + if( sqlite3_get_autocommit(p->db)==0 ){ + sqlite3_exec(p->db, "ROLLBACK", 0, 0, 0); + } + nBusy++; + rc = SQLITE_OK; + break; + } + else if( rc!=SQLITE_OK ){ + send_message(p, "error - %s (eec=%d)\n", sqlite3_errmsg(p->db), + sqlite3_extended_errcode(p->db) + ); + } + } + + t2 = get_timer(); + if( t2>=(t1+1000) ){ + int nMs = (t2 - t1); + int nDone = (j+1 - nBusy - nT1); + + rc = send_message( + p, "(%d done @ %d per second, %d busy)\n", + nDone, (1000*nDone + nMs/2) / nMs, nBusy - nTBusy1 + ); + t1 = t2; + nT1 = j+1 - nBusy; + nTBusy1 = nBusy; + if( p->nSecond>0 && (p->nSecond*1000)<=t1-t0 ) break; + } + + /* Global checkpoint handling. */ + if( g.nThreshold>0 ){ + pthread_mutex_lock(&g.ckpt_mutex); + if( rc==SQLITE_OK && g.bCkptRequired ){ + if( g.nWait==g.nRun-1 ){ + /* All other clients are already waiting on the condition variable. + ** Run the checkpoint, signal the condition and move on. */ + rc = sqlite3_wal_checkpoint(p->db, "main"); + g.bCkptRequired = 0; + pthread_cond_broadcast(&g.ckpt_cond); + }else{ + assert( g.nWaitbClientCkptRequired ){ + rc = sqlite3_wal_checkpoint(p->db, "main"); + assert( rc==SQLITE_OK ); + p->bClientCkptRequired = 0; + } + } + + if( rc==SQLITE_OK ){ + int nMs = (int)(get_timer() - t0); + send_message(p, "ok (%d/%d SQLITE_BUSY)\n", nBusy, j); + if( p->nRepeat<=0 ){ + send_message(p, "### ok %d busy %d ms %d\n", j-nBusy, nBusy, nMs); + } + } + clear_sql(p); + + pthread_mutex_lock(&g.ckpt_mutex); + g.nRun--; + pthread_mutex_unlock(&g.ckpt_mutex); + + return rc; +} + +static int handle_dot_command(ClientCtx *p, const char *zCmd, int nCmd){ + int n; + int rc = 0; + const char *z = &zCmd[1]; + const char *zArg; + int nArg; + + assert( zCmd[0]=='.' ); + for(n=0; n<(nCmd-1); n++){ + if( is_whitespace(z[n]) ) break; + } + + zArg = &z[n]; + nArg = nCmd-n; + trim_string(&zArg, &nArg); + + if( n>=1 && n<=4 && 0==strncmp(z, "list", n) ){ + int i; + for(i=0; rc==0 && inPrepare; i++){ + const char *zSql = sqlite3_sql(p->aPrepare[i].pStmt); + int nSql = strlen(zSql); + trim_string(&zSql, &nSql); + rc = send_message(p, "%d: %.*s\n", i, nSql, zSql); + } + } + + else if( n>=1 && n<=4 && 0==strncmp(z, "quit", n) ){ + rc = 1; + } + + else if( n>=2 && n<=7 && 0==strncmp(z, "repeats", n) ){ + if( nArg ){ + p->nRepeat = strtol(zArg, 0, 0); + if( p->nRepeat>0 ) p->nSecond = 0; + } + rc = send_message(p, "ok (repeat=%d)\n", p->nRepeat); + } + + else if( n>=2 && n<=3 && 0==strncmp(z, "run", n) ){ + rc = handle_run_command(p); + } + + else if( n>=2 && n<=7 && 0==strncmp(z, "seconds", n) ){ + if( nArg ){ + p->nSecond = strtol(zArg, 0, 0); + if( p->nSecond>0 ) p->nRepeat = 0; + } + rc = send_message(p, "ok (seconds=%d)\n", p->nSecond); + } + + else if( n>=1 && n<=12 && 0==strncmp(z, "mutex_commit", n) ){ + rc = handle_some_sql(p, "COMMIT;", 7); + if( rc==SQLITE_OK ){ + p->aPrepare[p->nPrepare-1].bMutex = 1; + } + } + + else if( n>=1 && n<=10 && 0==strncmp(z, "checkpoint", n) ){ + if( nArg ){ + p->nClientThreshold = strtol(zArg, 0, 0); + } + rc = send_message(p, "ok (checkpoint=%d)\n", p->nClientThreshold); + } + + else if( n>=2 && n<=4 && 0==strncmp(z, "stop", n) ){ + sqlite3_close(g.db); + exit(0); + } + + else{ + send_message(p, + "unrecognized dot command: %.*s\n" + "should be \"list\", \"run\", \"repeats\", \"mutex_commit\", " + "\"checkpoint\" or \"seconds\"\n", n, z + ); + rc = 1; + } + + return rc; +} + +static void *handle_client(void *pArg){ + char zCmd[32*1024]; /* Read buffer */ + int nCmd = 0; /* Valid bytes in zCmd[] */ + int res; /* Result of read() call */ + int rc = SQLITE_OK; + + ClientCtx ctx; + memset(&ctx, 0, sizeof(ClientCtx)); + + ctx.fd = (int)(intptr_t)pArg; + ctx.nRepeat = 1; + rc = sqlite3_open_v2(g.zDatabaseName, &ctx.db, + SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, g.zVfs + ); + if( rc!=SQLITE_OK ){ + fprintf(stderr, "sqlite3_open(): %s\n", sqlite3_errmsg(ctx.db)); + return 0; + } + sqlite3_create_function( + ctx.db, "usleep", 1, SQLITE_UTF8, (void*)sqlite3_vfs_find(0), + usleepFunc, 0, 0 + ); + + /* Register the wal-hook with the new client connection */ + sqlite3_wal_hook(ctx.db, clientWalHook, (void*)&ctx); + + while( rc==SQLITE_OK ){ + int i; + int iStart; + int nConsume; + res = read(ctx.fd, &zCmd[nCmd], sizeof(zCmd)-nCmd-1); + if( res<=0 ) break; + nCmd += res; + if( nCmd>=sizeof(zCmd)-1 ){ + fprintf(stderr, "oversized (>32KiB) message\n"); + res = 0; + break; + } + zCmd[nCmd] = '\0'; + + do { + nConsume = 0; + + /* Gobble up any whitespace */ + iStart = 0; + while( is_whitespace(zCmd[iStart]) ) iStart++; + + if( zCmd[iStart]=='.' ){ + /* This is a dot-command. Search for end-of-line. */ + for(i=iStart; i0 ){ + nCmd = nCmd-nConsume; + if( nCmd>0 ){ + memmove(zCmd, &zCmd[nConsume], nCmd); + } + } + }while( rc==SQLITE_OK && nConsume>0 ); + } + + fprintf(stdout, "Client %d disconnects\n", ctx.fd); + fflush(stdout); + close(ctx.fd); + clear_sql(&ctx); + sqlite3_free(ctx.aPrepare); + sqlite3_close(ctx.db); + return 0; +} + +static void usage(const char *zExec){ + fprintf(stderr, "Usage: %s ?-vfs VFS? DATABASE\n", zExec); + exit(1); +} + +int main(int argc, char *argv[]) { + int sfd; + int rc; + int yes = 1; + struct sockaddr_in server; + int i; + + /* Ignore SIGPIPE. Otherwise the server exits if a client disconnects + ** abruptly. */ + signal(SIGPIPE, SIG_IGN); + + g.nThreshold = TSERVER_DEFAULT_CHECKPOINT_THRESHOLD; + if( (argc%2) ) usage(argv[0]); + for(i=1; i<(argc-1); i+=2){ + int n = strlen(argv[i]); + if( n>=2 && 0==sqlite3_strnicmp("-walautocheckpoint", argv[i], n) ){ + g.nThreshold = strtol(argv[i+1], 0, 0); + }else + if( n>=2 && 0==sqlite3_strnicmp("-vfs", argv[i], n) ){ + g.zVfs = argv[i+1]; + } + } + g.zDatabaseName = argv[argc-1]; + + g.commit_mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + pthread_mutex_init(&g.ckpt_mutex, 0); + pthread_cond_init(&g.ckpt_cond, 0); + + rc = sqlite3_open_v2(g.zDatabaseName, &g.db, + SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, g.zVfs + ); + if( rc!=SQLITE_OK ){ + fprintf(stderr, "sqlite3_open(): %s\n", sqlite3_errmsg(g.db)); + return 1; + } + + rc = sqlite3_exec(g.db, "SELECT * FROM sqlite_master", 0, 0, 0); + if( rc!=SQLITE_OK ){ + fprintf(stderr, "sqlite3_exec(): %s\n", sqlite3_errmsg(g.db)); + return 1; + } + + sfd = socket(AF_INET, SOCK_STREAM, 0); + if( sfd<0 ){ + fprintf(stderr, "socket() failed\n"); + return 1; + } + + rc = setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); + if( rc<0 ){ + perror("setsockopt"); + return 1; + } + + memset(&server, 0, sizeof(server)); + server.sin_family = AF_INET; + server.sin_addr.s_addr = inet_addr("127.0.0.1"); + server.sin_port = htons(TSERVER_PORTNUMBER); + + rc = bind(sfd, (struct sockaddr *)&server, sizeof(struct sockaddr)); + if( rc<0 ){ + fprintf(stderr, "bind() failed\n"); + return 1; + } + + rc = listen(sfd, 8); + if( rc<0 ){ + fprintf(stderr, "listen() failed\n"); + return 1; + } + + while( 1 ){ + pthread_t tid; + int cfd = accept(sfd, NULL, NULL); + if( cfd<0 ){ + perror("accept()"); + return 1; + } + + fprintf(stdout, "Client %d connects\n", cfd); + fflush(stdout); + rc = pthread_create(&tid, NULL, handle_client, (void*)(intptr_t)cfd); + if( rc!=0 ){ + perror("pthread_create()"); + return 1; + } + + pthread_detach(tid); + } + + return 0; +} diff --git a/tool/tserver_test.tcl b/tool/tserver_test.tcl new file mode 100644 index 0000000000..39dec56b3c --- /dev/null +++ b/tool/tserver_test.tcl @@ -0,0 +1,283 @@ +#!/usr/bin/tclsh +# +# This script is used to run the performance test cases described in +# README-server-edition.html. +# + + +package require sqlite3 + +# Default values for command line switches: +set O(-database) "" +set O(-rows) [expr 5000000] +set O(-mode) wal2 +set O(-tserver) "./tserver" +set O(-seconds) 20 +set O(-writers) 1 +set O(-readers) 0 +set O(-verbose) 0 + + +proc error_out {err} { + puts stderr $err + exit -1 +} + +proc usage {} { + puts stderr "Usage: $::argv0 ?OPTIONS?" + puts stderr "" + puts stderr "Where OPTIONS are:" + puts stderr " -database (default: test.db)" + puts stderr " -mode (default: wal2)" + puts stderr " -rows (default: 5000000)" + puts stderr " -tserver (default: ./tserver)" + puts stderr " -seconds